Browse Source

HUE-9089 [hive] Upgrade Thrift protocol to V11 to support LLAP

As part of task support, LLAP requires HIVE_CLI_SERVICE_PROTOCOL_V8 and
changes the behavior of hasResultSet in TOperationHandle as it can be
updated in TGetOperationStatusResp. This is specific to LLAP as TEZ
keeps the old behavior of settings hasResultSet in
TExecuteStatementResp.
In further commits, we might want to also use TProgressUpdateResp that
came in as part of HIVE_CLI_SERVICE_PROTOCOL_V10 to track query progress
as an alternative to looking at Yarn for job progress.
Jean-Francois Desjeans Gauthier 6 năm trước cách đây
mục cha
commit
ebbca5722a

+ 28 - 0
apps/beeswax/gen-py/TCLIService/TCLIService-remote

@@ -35,6 +35,8 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print('  TGetTableTypesResp GetTableTypes(TGetTableTypesReq req)')
   print('  TGetColumnsResp GetColumns(TGetColumnsReq req)')
   print('  TGetFunctionsResp GetFunctions(TGetFunctionsReq req)')
+  print('  TGetPrimaryKeysResp GetPrimaryKeys(TGetPrimaryKeysReq req)')
+  print('  TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req)')
   print('  TGetOperationStatusResp GetOperationStatus(TGetOperationStatusReq req)')
   print('  TCancelOperationResp CancelOperation(TCancelOperationReq req)')
   print('  TCloseOperationResp CloseOperation(TCloseOperationReq req)')
@@ -43,6 +45,8 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print('  TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req)')
   print('  TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req)')
   print('  TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req)')
+  print('  TGetQueryIdResp GetQueryId(TGetQueryIdReq req)')
+  print('  TSetClientInfoResp SetClientInfo(TSetClientInfoReq req)')
   print('  TGetLogResp GetLog(TGetLogReq req)')
   print('')
   sys.exit(0)
@@ -166,6 +170,18 @@ elif cmd == 'GetFunctions':
     sys.exit(1)
   pp.pprint(client.GetFunctions(eval(args[0]),))
 
+elif cmd == 'GetPrimaryKeys':
+  if len(args) != 1:
+    print('GetPrimaryKeys requires 1 args')
+    sys.exit(1)
+  pp.pprint(client.GetPrimaryKeys(eval(args[0]),))
+
+elif cmd == 'GetCrossReference':
+  if len(args) != 1:
+    print('GetCrossReference requires 1 args')
+    sys.exit(1)
+  pp.pprint(client.GetCrossReference(eval(args[0]),))
+
 elif cmd == 'GetOperationStatus':
   if len(args) != 1:
     print('GetOperationStatus requires 1 args')
@@ -214,6 +230,18 @@ elif cmd == 'RenewDelegationToken':
     sys.exit(1)
   pp.pprint(client.RenewDelegationToken(eval(args[0]),))
 
+elif cmd == 'GetQueryId':
+  if len(args) != 1:
+    print('GetQueryId requires 1 args')
+    sys.exit(1)
+  pp.pprint(client.GetQueryId(eval(args[0]),))
+
+elif cmd == 'SetClientInfo':
+  if len(args) != 1:
+    print('SetClientInfo requires 1 args')
+    sys.exit(1)
+  pp.pprint(client.SetClientInfo(eval(args[0]),))
+
 elif cmd == 'GetLog':
   if len(args) != 1:
     print('GetLog requires 1 args')

Những thai đổi đã bị hủy bỏ vì nó quá lớn
+ 546 - 52
apps/beeswax/gen-py/TCLIService/TCLIService.py


+ 6 - 0
apps/beeswax/gen-py/TCLIService/constants.py

@@ -26,6 +26,9 @@ PRIMITIVE_TYPES = set([
     17,
     18,
     19,
+    20,
+    21,
+    22,
 ])
 COMPLEX_TYPES = set([
     10,
@@ -58,6 +61,9 @@ TYPE_NAMES = {
     17 : "DATE",
     18 : "VARCHAR",
     19 : "CHAR",
+    20 : "INTERVAL_YEAR_MONTH",
+    21 : "INTERVAL_DAY_TIME",
+    22 : "TIMESTAMP WITH LOCAL TIME ZONE",
 }
 CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength"
 PRECISION = "precision"

Những thai đổi đã bị hủy bỏ vì nó quá lớn
+ 898 - 42
apps/beeswax/gen-py/TCLIService/ttypes.py


+ 1 - 1
apps/beeswax/src/beeswax/conf.py

@@ -236,7 +236,7 @@ THRIFT_VERSION = Config(
   key="thrift_version",
   help=_t("Thrift version to use when communicating with HiveServer2."),
   type=int,
-  default=7
+  default=11
 )
 
 CONFIG_WHITELIST = Config(

+ 164 - 28
apps/beeswax/thrift/TCLIService.thrift

@@ -32,31 +32,43 @@
 // * Service names begin with the letter "T", use a capital letter for each
 //   new word (with no underscores), and end with the word "Service".
 
-namespace java org.apache.hive.service.cli.thrift
-namespace cpp apache.hive.service.cli.thrift
+namespace java org.apache.hive.service.rpc.thrift
+namespace cpp apache.hive.service.rpc.thrift
 
 // List of protocol versions. A new token should be
 // added to the end of this list every time a change is made.
 enum TProtocolVersion {
   HIVE_CLI_SERVICE_PROTOCOL_V1,
-  
+
   // V2 adds support for asynchronous execution
-  HIVE_CLI_SERVICE_PROTOCOL_V2,
+  HIVE_CLI_SERVICE_PROTOCOL_V2
 
   // V3 add varchar type, primitive type qualifiers
-  HIVE_CLI_SERVICE_PROTOCOL_V3,
+  HIVE_CLI_SERVICE_PROTOCOL_V3
 
   // V4 add decimal precision/scale, char type
-  HIVE_CLI_SERVICE_PROTOCOL_V4,
+  HIVE_CLI_SERVICE_PROTOCOL_V4
 
   // V5 adds error details when GetOperationStatus returns in error state
-  HIVE_CLI_SERVICE_PROTOCOL_V5,
+  HIVE_CLI_SERVICE_PROTOCOL_V5
 
   // V6 uses binary type for binary payload (was string) and uses columnar result set
-  HIVE_CLI_SERVICE_PROTOCOL_V6,
+  HIVE_CLI_SERVICE_PROTOCOL_V6
 
   // V7 adds support for delegation token based connection
   HIVE_CLI_SERVICE_PROTOCOL_V7
+
+  // V8 adds support for interval types
+  HIVE_CLI_SERVICE_PROTOCOL_V8
+
+  // V9 adds support for serializing ResultSets in SerDe
+  HIVE_CLI_SERVICE_PROTOCOL_V9
+
+  // V10 adds support for in place updates via GetOperationStatus
+  HIVE_CLI_SERVICE_PROTOCOL_V10
+
+  // V11 adds timestamp with local time zone type
+  HIVE_CLI_SERVICE_PROTOCOL_V11
 }
 
 enum TTypeId {
@@ -79,9 +91,12 @@ enum TTypeId {
   NULL_TYPE,
   DATE_TYPE,
   VARCHAR_TYPE,
-  CHAR_TYPE
+  CHAR_TYPE,
+  INTERVAL_YEAR_MONTH_TYPE,
+  INTERVAL_DAY_TIME_TYPE,
+  TIMESTAMPLOCALTZ_TYPE
 }
-  
+
 const set<TTypeId> PRIMITIVE_TYPES = [
   TTypeId.BOOLEAN_TYPE,
   TTypeId.TINYINT_TYPE,
@@ -97,7 +112,10 @@ const set<TTypeId> PRIMITIVE_TYPES = [
   TTypeId.NULL_TYPE,
   TTypeId.DATE_TYPE,
   TTypeId.VARCHAR_TYPE,
-  TTypeId.CHAR_TYPE
+  TTypeId.CHAR_TYPE,
+  TTypeId.INTERVAL_YEAR_MONTH_TYPE,
+  TTypeId.INTERVAL_DAY_TIME_TYPE,
+  TTypeId.TIMESTAMPLOCALTZ_TYPE
 ]
 
 const set<TTypeId> COMPLEX_TYPES = [
@@ -133,6 +151,9 @@ const map<TTypeId,string> TYPE_NAMES = {
   TTypeId.DATE_TYPE: "DATE"
   TTypeId.VARCHAR_TYPE: "VARCHAR"
   TTypeId.CHAR_TYPE: "CHAR"
+  TTypeId.INTERVAL_YEAR_MONTH_TYPE: "INTERVAL_YEAR_MONTH"
+  TTypeId.INTERVAL_DAY_TIME_TYPE: "INTERVAL_DAY_TIME"
+  TTypeId.TIMESTAMPLOCALTZ_TYPE: "TIMESTAMP WITH LOCAL TIME ZONE"
 }
 
 // Thrift does not support recursively defined types or forward declarations,
@@ -256,7 +277,7 @@ struct TColumnDesc {
 
   // The type descriptor for this column
   2: required TTypeDesc typeDesc
-  
+
   // The ordinal position of this column in the schema
   3: required i32 position
 
@@ -323,7 +344,7 @@ union TColumnValue {
   4: TI32Value    i32Val       // INT
   5: TI64Value    i64Val       // BIGINT, TIMESTAMP
   6: TDoubleValue doubleVal    // FLOAT, DOUBLE
-  7: TStringValue stringVal    // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY, DECIMAL, NULL
+  7: TStringValue stringVal    // STRING, LIST, MAP, STRUCT, UNIONTYPE, BINARY, DECIMAL, NULL, INTERVAL_YEAR_MONTH, INTERVAL_DAY_TIME
 }
 
 // Represents a row in a rowset.
@@ -393,6 +414,8 @@ struct TRowSet {
   1: required i64 startRowOffset
   2: required list<TRow> rows
   3: optional list<TColumn> columns
+  4: optional binary binaryColumns
+  5: optional i32 columnCount
 }
 
 // The return status code contained in each response.
@@ -447,6 +470,9 @@ enum TOperationState {
 
   // The operation is in an pending state
   PENDING_STATE,
+
+  // The operation is in an timedout state
+  TIMEDOUT_STATE,
 }
 
 // A string identifier. This is interpreted literally.
@@ -542,7 +568,7 @@ struct TOperationHandle {
 // which operations may be executed.
 struct TOpenSessionReq {
   // The version of the HiveServer2 protocol that the client is using.
-  1: required TProtocolVersion client_protocol = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6
+  1: required TProtocolVersion client_protocol = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10
 
   // Username and password for authentication.
   // Depending on the authentication scheme being used,
@@ -561,7 +587,7 @@ struct TOpenSessionResp {
   1: required TStatus status
 
   // The protocol version that the server is using.
-  2: required TProtocolVersion serverProtocolVersion = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6
+  2: required TProtocolVersion serverProtocolVersion = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10
 
   // Session Handle
   3: optional TSessionHandle sessionHandle
@@ -570,6 +596,15 @@ struct TOpenSessionResp {
   4: optional map<string, string> configuration
 }
 
+struct TSetClientInfoReq {
+  1: required TSessionHandle sessionHandle,
+  2: optional map<string, string> configuration
+}
+
+struct TSetClientInfoResp {
+  1: required TStatus status
+}
+
 
 // CloseSession()
 //
@@ -635,6 +670,7 @@ enum TGetInfoType {
   CLI_CATALOG_NAME =                     10003,
   CLI_COLLATION_SEQ =                    10004,
   CLI_MAX_IDENTIFIER_LEN =               10005,
+  CLI_ODBC_KEYWORDS =                    10006
 }
 
 union TGetInfoValue {
@@ -683,9 +719,12 @@ struct TExecuteStatementReq {
   // is executed. These properties apply to this statement
   // only and will not affect the subsequent state of the Session.
   3: optional map<string, string> confOverlay
-  
+
   // Execute asynchronously when runAsync is true
   4: optional bool runAsync = false
+
+  // The number of seconds after which the query will timeout on the server
+  5: optional i64 queryTimeout = 0
 }
 
 struct TExecuteStatementResp {
@@ -709,13 +748,13 @@ struct TGetTypeInfoReq {
 struct TGetTypeInfoResp {
   1: required TStatus status
   2: optional TOperationHandle operationHandle
-}  
+}
 
 
 // GetCatalogs()
 //
-// Returns the list of catalogs (databases) 
-// Results are ordered by TABLE_CATALOG 
+// Returns the list of catalogs (databases)
+// Results are ordered by TABLE_CATALOG
 //
 // Resultset columns :
 // col1
@@ -825,9 +864,9 @@ struct TGetTablesResp {
 
 // GetTableTypes()
 //
-// Returns the table types available in this database. 
-// The results are ordered by table type. 
-// 
+// Returns the table types available in this database.
+// The results are ordered by table type.
+//
 // col1
 // name: TABLE_TYPE
 // type: STRING
@@ -848,8 +887,8 @@ struct TGetTableTypesResp {
 // Returns a list of columns in the specified tables.
 // The information is returned as a result set which can be fetched
 // using the OperationHandle provided in the response.
-// Results are ordered by TABLE_CAT, TABLE_SCHEM, TABLE_NAME, 
-// and ORDINAL_POSITION. 
+// Results are ordered by TABLE_CAT, TABLE_SCHEM, TABLE_NAME,
+// and ORDINAL_POSITION.
 //
 // Result Set Columns are the same as those for the ODBC CLIColumns
 // function.
@@ -945,7 +984,53 @@ struct TGetFunctionsResp {
   1: required TStatus status
   2: optional TOperationHandle operationHandle
 }
-  
+
+struct TGetPrimaryKeysReq {
+  // Session to run this request against
+  1: required TSessionHandle sessionHandle
+
+  // Name of the catalog.
+  2: optional TIdentifier catalogName
+
+  // Name of the schema.
+  3: optional TIdentifier schemaName
+
+  // Name of the table.
+  4: optional TIdentifier tableName
+}
+
+struct TGetPrimaryKeysResp {
+  1: required TStatus status
+  2: optional TOperationHandle operationHandle
+}
+
+struct TGetCrossReferenceReq {
+  // Session to run this request against
+  1: required TSessionHandle sessionHandle
+
+  // Name of the parent catalog.
+  2: optional TIdentifier parentCatalogName
+
+  // Name of the parent schema.
+  3: optional TIdentifier parentSchemaName
+
+  // Name of the parent table.
+  4: optional TIdentifier parentTableName
+
+  // Name of the foreign catalog.
+  5: optional TIdentifier foreignCatalogName
+
+  // Name of the foreign schema.
+  6: optional TIdentifier foreignSchemaName
+
+  // Name of the foreign table.
+  7: optional TIdentifier foreignTableName
+}
+
+struct TGetCrossReferenceResp {
+  1: required TStatus status
+  2: optional TOperationHandle operationHandle
+}
 
 // GetOperationStatus()
 //
@@ -953,6 +1038,18 @@ struct TGetFunctionsResp {
 struct TGetOperationStatusReq {
   // Session to run this request against
   1: required TOperationHandle operationHandle
+  // optional arguments to get progress information
+  2: optional bool getProgressUpdate
+}
+
+// TProgressUpdateResp was defined below, but it's used by TGetOperationStatusResp so had to be moved up
+struct TProgressUpdateResp {
+  1: required list<string> headerNames
+  2: required list<list<string>> rows
+  3: required double progressedPercentage
+  4: required TJobExecutionStatus status
+  5: required string footerSummary
+  6: required i64 startTime
 }
 
 struct TGetOperationStatusResp {
@@ -968,6 +1065,22 @@ struct TGetOperationStatusResp {
 
   // Error message
   5: optional string errorMessage
+
+  // List of statuses of sub tasks
+  6: optional string taskStatus
+
+  // When was the operation started
+  7: optional i64 operationStarted
+
+  // When was the operation completed
+  8: optional i64 operationCompleted
+
+  // If the operation has the result
+  9: optional bool hasResultSet
+
+  10: optional TProgressUpdateResp progressUpdateResponse
+
+  11: optional i64 numModifiedRows
 }
 
 
@@ -1050,7 +1163,7 @@ struct TFetchResultsReq {
   // The fetch orientation. For V1 this must be either
   // FETCH_NEXT or FETCH_FIRST. Defaults to FETCH_NEXT.
   2: required TFetchOrientation orientation = TFetchOrientation.FETCH_NEXT
-  
+
   // Max number of rows that should be returned in
   // the rowset.
   3: required i64 maxRows
@@ -1123,6 +1236,20 @@ struct TRenewDelegationTokenResp {
   1: required TStatus status
 }
 
+enum TJobExecutionStatus {
+    IN_PROGRESS,
+    COMPLETE,
+    NOT_AVAILABLE
+}
+
+struct TGetQueryIdReq {
+  1: required TOperationHandle operationHandle
+}
+
+struct TGetQueryIdResp {
+  1: required string queryId
+}
+
 // GetLog()
 //
 // Fetch operation log from the server corresponding to
@@ -1163,8 +1290,12 @@ service TCLIService {
 
   TGetFunctionsResp GetFunctions(1:TGetFunctionsReq req);
 
+  TGetPrimaryKeysResp GetPrimaryKeys(1:TGetPrimaryKeysReq req);
+
+  TGetCrossReferenceResp GetCrossReference(1:TGetCrossReferenceReq req);
+
   TGetOperationStatusResp GetOperationStatus(1:TGetOperationStatusReq req);
-  
+
   TCancelOperationResp CancelOperation(1:TCancelOperationReq req);
 
   TCloseOperationResp CloseOperation(1:TCloseOperationReq req);
@@ -1179,5 +1310,10 @@ service TCLIService {
 
   TRenewDelegationTokenResp RenewDelegationToken(1:TRenewDelegationTokenReq req);
 
+  TGetQueryIdResp GetQueryId(1:TGetQueryIdReq req);
+
+  TSetClientInfoResp SetClientInfo(1:TSetClientInfoReq req);
+
+  // Not present in Hive 0.13, re-added for backwards compatibility.
   TGetLogResp GetLog(1:TGetLogReq req);
-}
+}

+ 2 - 2
desktop/conf.dist/hue.ini

@@ -1246,8 +1246,8 @@
   ## max_number_of_sessions=1
 
   # Thrift version to use when communicating with HiveServer2.
-  # New column format is from version 7.
-  ## thrift_version=7
+  # New column format is from version 11.
+  ## thrift_version=11
 
   # A comma-separated list of white-listed Hive configuration properties that users are authorized to set.
   ## config_whitelist=hive.map.aggr,hive.exec.compress.output,hive.exec.parallel,hive.execution.engine,mapreduce.job.queuename

+ 2 - 2
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1230,8 +1230,8 @@
   ## max_number_of_sessions=1
 
   # Thrift version to use when communicating with HiveServer2.
-  # New column format is from version 7.
-  ## thrift_version=7
+  # New column format is from version 11.
+  ## thrift_version=11
 
   # A comma-separated list of white-listed Hive configuration properties that users are authorized to set.
   ## config_whitelist=hive.map.aggr,hive.exec.compress.output,hive.exec.parallel,hive.execution.engine,mapreduce.job.queuename

Một số tệp đã không được hiển thị bởi vì quá nhiều tập tin thay đổi trong này khác