Преглед на файлове

HUE-8737 [core] Fix Hive Thrift (step 2) for Python 3.5

Ying Chen преди 6 години
родител
ревизия
d6a084828f

+ 26 - 6
apps/beeswax/gen-py/TCLIService/ttypes.py

@@ -6,6 +6,8 @@
 #  options string: py:new_style
 #
 
+import sys
+
 from builtins import range
 from builtins import object
 from thrift.Thrift import TType, TMessageType, TException, TApplicationException
@@ -2646,7 +2648,10 @@ class TStringColumn(object):
           iprot.skip(ftype)
       elif fid == 2:
         if ftype == TType.STRING:
-          self.nulls = iprot.readString()
+          if sys.version_info[0] > 2:
+            self.nulls = iprot.readBinary()
+          else:
+            self.nulls = iprot.readString()
         else:
           iprot.skip(ftype)
       else:
@@ -3220,12 +3225,18 @@ class THandleIdentifier(object):
         break
       if fid == 1:
         if ftype == TType.STRING:
-          self.guid = iprot.readString()
+          if sys.version_info[0] > 2:
+            self.guid = iprot.readBinary()
+          else:
+            self.guid = iprot.readString()
         else:
           iprot.skip(ftype)
       elif fid == 2:
         if ftype == TType.STRING:
-          self.secret = iprot.readString()
+          if sys.version_info[0] > 2:
+            self.secret = iprot.readBinary()
+          else:
+            self.secret = iprot.readString()
         else:
           iprot.skip(ftype)
       else:
@@ -3240,11 +3251,17 @@ class THandleIdentifier(object):
     oprot.writeStructBegin('THandleIdentifier')
     if self.guid is not None:
       oprot.writeFieldBegin('guid', TType.STRING, 1)
-      oprot.writeString(self.guid)
+      if sys.version_info[0] > 2:
+        oprot.writeBinary(self.guid)
+      else:
+        oprot.writeString(self.guid)
       oprot.writeFieldEnd()
     if self.secret is not None:
       oprot.writeFieldBegin('secret', TType.STRING, 2)
-      oprot.writeString(self.secret)
+      if sys.version_info[0] > 2:
+        oprot.writeBinary(self.secret)
+      else:
+        oprot.writeString(self.secret)
       oprot.writeFieldEnd()
     oprot.writeFieldStop()
     oprot.writeStructEnd()
@@ -4197,7 +4214,10 @@ class TExecuteStatementReq(object):
       oprot.writeFieldEnd()
     if self.statement is not None:
       oprot.writeFieldBegin('statement', TType.STRING, 2)
-      oprot.writeString(self.statement)
+      if sys.version_info[0] > 2:
+        oprot.writeBinary(self.statement)
+      else:
+        oprot.writeString(self.statement)
       oprot.writeFieldEnd()
     if self.confOverlay is not None:
       oprot.writeFieldBegin('confOverlay', TType.MAP, 3)

+ 4 - 1
apps/beeswax/src/beeswax/models.py

@@ -488,7 +488,10 @@ class HiveServerQueryHandle(QueryHandle):
       return base64.decodestring(secret), base64.decodestring(guid)
 
   def get_encoded(self):
-    return base64.encodestring(self.secret), base64.encodestring(self.guid)
+    if sys.version_info[0] > 2:
+      return base64.b64encode(self.secret), base64.b64encode(self.guid)
+    else:
+      return base64.encodestring(self.secret), base64.encodestring(self.guid)
 
 
 # Deprecated. Could be removed.

+ 13 - 3
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -18,12 +18,12 @@
 from builtins import next
 from builtins import filter
 from builtins import map
-from builtins import str
 from builtins import object
 import logging
 import itertools
 import json
 import re
+import sys
 
 
 from operator import itemgetter
@@ -295,7 +295,10 @@ class HiveServerTColumnValue2(object):
 
   @classmethod
   def mark_nulls(cls, values, bytestring):
-    mask = bytearray(bytestring)
+    if sys.version_info[0] < 3 or isinstance(bytestring, bytes):
+      mask = bytearray(bytestring)
+    else:
+      mask = bytearray(bytestring, 'utf-8')
 
     for n in mask:
       yield n & 0x01
@@ -310,7 +313,14 @@ class HiveServerTColumnValue2(object):
 
   @classmethod
   def set_nulls(cls, values, bytestring):
-    if bytestring == '' or re.match('^(\x00)+$', bytestring): # HS2 has just \x00 or '', Impala can have \x00\x00...
+    can_decode = True
+    if sys.version_info[0] == 3 and isinstance(bytestring, bytes):
+      try:
+        bytestring = bytestring.decode('utf-8')
+      except:
+        can_decode = False
+
+    if bytestring == '' or (can_decode and re.match('^(\x00)+$', bytestring)): # HS2 has just \x00 or '', Impala can have \x00\x00...
       return values
     else:
       _values = [None if is_null else value for value, is_null in zip(values, cls.mark_nulls(values, bytestring))]

+ 1 - 1
desktop/core/src/desktop/lib/thrift_sasl.py

@@ -46,7 +46,7 @@ class TSaslClientTransport(TTransportBase, CReadableTransport):
   def __init__(self, sasl_client_factory, mechanism, trans):
     """
     @param sasl_client_factory: a callable that returns a new sasl.Client object
-    @param mechanism: the SASL mechanism (e.g. "GSSAPI")
+    @param mechanism: the SASL mechanism (e.g. "GSSAPI", "PLAIN")
     @param trans: the underlying transport over which to communicate.
     """
     self._trans = trans

+ 5 - 0
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -26,6 +26,7 @@ import copy
 import json
 import logging
 import re
+import struct
 import sys
 
 from django.urls import reverse
@@ -270,6 +271,10 @@ class HS2Api(Api):
 
     # All good
     server_id, server_guid = handle.get()
+    if sys.version_info[0] > 2:
+      server_id = server_id.decode('utf-8')
+      server_guid = server_guid.decode('utf-8')
+
     response = {
       'secret': server_id,
       'guid': server_guid,

+ 4 - 1
desktop/libs/notebook/src/notebook/sql_utils.py

@@ -84,7 +84,10 @@ def get_current_statement(snippet):
 
 
 def compute_statement_hash(statement):
-  return hashlib.sha224(smart_str(statement)).hexdigest()
+  if sys.version_info[0] > 2:
+    return hashlib.sha224(statement.encode()).hexdigest()
+  else:
+    return hashlib.sha224(smart_str(statement)).hexdigest()
 
 def split_statements(hql):
   """