|
|
@@ -44,10 +44,66 @@ struct Type {
|
|
|
4: optional list<FieldSchema> fields // if the name is one of the user defined types
|
|
|
}
|
|
|
|
|
|
+enum HiveObjectType {
|
|
|
+ GLOBAL = 1,
|
|
|
+ DATABASE = 2,
|
|
|
+ TABLE = 3,
|
|
|
+ PARTITION = 4,
|
|
|
+ COLUMN = 5,
|
|
|
+}
|
|
|
+
|
|
|
+enum PrincipalType {
|
|
|
+ USER = 1,
|
|
|
+ ROLE = 2,
|
|
|
+ GROUP = 3,
|
|
|
+}
|
|
|
+
|
|
|
+struct HiveObjectRef{
|
|
|
+ 1: HiveObjectType objectType,
|
|
|
+ 2: string dbName,
|
|
|
+ 3: string objectName,
|
|
|
+ 4: list<string> partValues,
|
|
|
+ 5: string columnName,
|
|
|
+}
|
|
|
+
|
|
|
+struct PrivilegeGrantInfo {
|
|
|
+ 1: string privilege,
|
|
|
+ 2: i32 createTime,
|
|
|
+ 3: string grantor,
|
|
|
+ 4: PrincipalType grantorType,
|
|
|
+ 5: bool grantOption,
|
|
|
+}
|
|
|
+
|
|
|
+struct HiveObjectPrivilege {
|
|
|
+ 1: HiveObjectRef hiveObject,
|
|
|
+ 2: string principalName,
|
|
|
+ 3: PrincipalType principalType,
|
|
|
+ 4: PrivilegeGrantInfo grantInfo,
|
|
|
+}
|
|
|
+
|
|
|
+struct PrivilegeBag {
|
|
|
+ 1: list<HiveObjectPrivilege> privileges,
|
|
|
+}
|
|
|
+
|
|
|
+struct PrincipalPrivilegeSet {
|
|
|
+ 1: map<string, list<PrivilegeGrantInfo>> userPrivileges, // user name -> privilege grant info
|
|
|
+ 2: map<string, list<PrivilegeGrantInfo>> groupPrivileges, // group name -> privilege grant info
|
|
|
+ 3: map<string, list<PrivilegeGrantInfo>> rolePrivileges, //role name -> privilege grant info
|
|
|
+}
|
|
|
+
|
|
|
+struct Role {
|
|
|
+ 1: string roleName,
|
|
|
+ 2: i32 createTime,
|
|
|
+ 3: string ownerName,
|
|
|
+}
|
|
|
+
|
|
|
// namespace for tables
|
|
|
struct Database {
|
|
|
1: string name,
|
|
|
2: string description,
|
|
|
+ 3: string locationUri,
|
|
|
+ 4: map<string, string> parameters, // properties associated with the database
|
|
|
+ 5: optional PrincipalPrivilegeSet privileges
|
|
|
}
|
|
|
|
|
|
// This object holds the information needed by SerDes
|
|
|
@@ -87,7 +143,11 @@ struct Table {
|
|
|
6: i32 retention, // retention time
|
|
|
7: StorageDescriptor sd, // storage descriptor of the table
|
|
|
8: list<FieldSchema> partitionKeys, // partition keys of the table. only primitive types are supported
|
|
|
- 9: map<string, string> parameters // to store comments or any other user level parameters
|
|
|
+ 9: map<string, string> parameters, // to store comments or any other user level parameters
|
|
|
+ 10: string viewOriginalText, // original view text, null for non-view
|
|
|
+ 11: string viewExpandedText, // expanded view text, null for non-view
|
|
|
+ 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
|
|
|
+ 13: optional PrincipalPrivilegeSet privileges,
|
|
|
}
|
|
|
|
|
|
struct Partition {
|
|
|
@@ -97,18 +157,21 @@ struct Partition {
|
|
|
4: i32 createTime,
|
|
|
5: i32 lastAccessTime,
|
|
|
6: StorageDescriptor sd,
|
|
|
- 7: map<string, string> parameters
|
|
|
+ 7: map<string, string> parameters,
|
|
|
+ 8: optional PrincipalPrivilegeSet privileges
|
|
|
}
|
|
|
|
|
|
-// index on a hive table is also another table whose columns are the subset of the base table columns along with the offset
|
|
|
-// this will automatically generate table (table_name_index_name)
|
|
|
struct Index {
|
|
|
1: string indexName, // unique with in the whole database namespace
|
|
|
- 2: i32 indexType, // reserved
|
|
|
- 3: string tableName,
|
|
|
- 4: string dbName,
|
|
|
- 5: list<string> colNames, // for now columns will be sorted in the ascending order
|
|
|
- 6: string partName // partition name
|
|
|
+ 2: string indexHandlerClass, // reserved
|
|
|
+ 3: string dbName,
|
|
|
+ 4: string origTableName,
|
|
|
+ 5: i32 createTime,
|
|
|
+ 6: i32 lastAccessTime,
|
|
|
+ 7: string indexTableName,
|
|
|
+ 8: StorageDescriptor sd,
|
|
|
+ 9: map<string, string> parameters,
|
|
|
+ 10: bool deferredRebuild
|
|
|
}
|
|
|
|
|
|
// schema of the table/query results etc.
|
|
|
@@ -118,7 +181,6 @@ struct Schema {
|
|
|
2: map<string, string> properties
|
|
|
}
|
|
|
|
|
|
-
|
|
|
exception MetaException {
|
|
|
1: string message
|
|
|
}
|
|
|
@@ -160,16 +222,17 @@ exception ConfigValSecurityException {
|
|
|
*/
|
|
|
service ThriftHiveMetastore extends fb303.FacebookService
|
|
|
{
|
|
|
- bool create_database(1:string name, 2:string description)
|
|
|
- throws(1:AlreadyExistsException o1, 2:MetaException o2)
|
|
|
+ void create_database(1:Database database) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
|
|
|
Database get_database(1:string name) throws(1:NoSuchObjectException o1, 2:MetaException o2)
|
|
|
- bool drop_database(1:string name) throws(2:MetaException o2)
|
|
|
- list<string> get_databases() throws(1:MetaException o1)
|
|
|
-
|
|
|
+ void drop_database(1:string name, 2:bool deleteData) throws(1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3)
|
|
|
+ list<string> get_databases(1:string pattern) throws(1:MetaException o1)
|
|
|
+ list<string> get_all_databases() throws(1:MetaException o1)
|
|
|
+ void alter_database(1:string dbname, 2:Database db) throws(1:MetaException o1, 2:NoSuchObjectException o2)
|
|
|
+
|
|
|
// returns the type with given name (make seperate calls for the dependent types if needed)
|
|
|
- Type get_type(1:string name) throws(1:MetaException o2)
|
|
|
+ Type get_type(1:string name) throws(1:MetaException o1, 2:NoSuchObjectException o2)
|
|
|
bool create_type(1:Type type) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
|
|
|
- bool drop_type(1:string type) throws(1:MetaException o2)
|
|
|
+ bool drop_type(1:string type) throws(1:MetaException o1, 2:NoSuchObjectException o2)
|
|
|
map<string, Type> get_type_all(1:string name)
|
|
|
throws(1:MetaException o2)
|
|
|
|
|
|
@@ -187,38 +250,77 @@ service ThriftHiveMetastore extends fb303.FacebookService
|
|
|
// sd.inputFormat (SequenceFileInputFormat (binary like falcon tables or u_full) or TextInputFormat)
|
|
|
// sd.outputFormat (SequenceFileInputFormat (binary) or TextInputFormat)
|
|
|
// sd.serdeInfo.serializationLib (SerDe class name eg org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe
|
|
|
+ // * See notes on DDL_TIME
|
|
|
void create_table(1:Table tbl) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:NoSuchObjectException o4)
|
|
|
// drops the table and all the partitions associated with it if the table has partitions
|
|
|
// delete data (including partitions) if deleteData is set to true
|
|
|
void drop_table(1:string dbname, 2:string name, 3:bool deleteData)
|
|
|
throws(1:NoSuchObjectException o1, 2:MetaException o3)
|
|
|
- list<string> get_tables(1: string db_name, 2: string pattern)
|
|
|
- throws (1: MetaException o1)
|
|
|
+ list<string> get_tables(1: string db_name, 2: string pattern) throws (1: MetaException o1)
|
|
|
+ list<string> get_all_tables(1: string db_name) throws (1: MetaException o1)
|
|
|
|
|
|
Table get_table(1:string dbname, 2:string tbl_name)
|
|
|
throws (1:MetaException o1, 2:NoSuchObjectException o2)
|
|
|
// alter table applies to only future partitions not for existing partitions
|
|
|
+ // * See notes on DDL_TIME
|
|
|
void alter_table(1:string dbname, 2:string tbl_name, 3:Table new_tbl)
|
|
|
throws (1:InvalidOperationException o1, 2:MetaException o2)
|
|
|
|
|
|
// the following applies to only tables that have partitions
|
|
|
+ // * See notes on DDL_TIME
|
|
|
Partition add_partition(1:Partition new_part)
|
|
|
throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
|
|
|
Partition append_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
|
|
|
throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
|
|
|
+ Partition append_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name)
|
|
|
+ throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
|
|
|
bool drop_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:bool deleteData)
|
|
|
throws(1:NoSuchObjectException o1, 2:MetaException o2)
|
|
|
+ bool drop_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name, 4:bool deleteData)
|
|
|
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
|
|
|
Partition get_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
|
|
|
- throws(1:MetaException o1)
|
|
|
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
|
|
|
+
|
|
|
+ Partition get_partition_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals,
|
|
|
+ 4: string user_name, 5: list<string> group_names) throws(1:MetaException o1, 2:NoSuchObjectException o2)
|
|
|
+
|
|
|
+ Partition get_partition_by_name(1:string db_name 2:string tbl_name, 3:string part_name)
|
|
|
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
|
|
|
+
|
|
|
// returns all the partitions for this table in reverse chronological order.
|
|
|
- // if max parts is given then it will return only that many
|
|
|
+ // If max parts is given then it will return only that many.
|
|
|
list<Partition> get_partitions(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
|
|
|
throws(1:NoSuchObjectException o1, 2:MetaException o2)
|
|
|
+ list<Partition> get_partitions_with_auth(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1,
|
|
|
+ 4: string user_name, 5: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
|
|
|
+
|
|
|
list<string> get_partition_names(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
|
|
|
throws(1:MetaException o2)
|
|
|
+
|
|
|
+ // get_partition*_ps methods allow filtering by a partial partition specification,
|
|
|
+ // as needed for dynamic partitions. The values that are not restricted should
|
|
|
+ // be empty strings. Nulls were considered (instead of "") but caused errors in
|
|
|
+ // generated Python code. The size of part_vals may be smaller than the
|
|
|
+ // number of partition columns - the unspecified values are considered the same
|
|
|
+ // as "".
|
|
|
+ list<Partition> get_partitions_ps(1:string db_name 2:string tbl_name
|
|
|
+ 3:list<string> part_vals, 4:i16 max_parts=-1)
|
|
|
+ throws(1:MetaException o1)
|
|
|
+ list<Partition> get_partitions_ps_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1,
|
|
|
+ 5: string user_name, 6: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
|
|
|
+
|
|
|
+ list<string> get_partition_names_ps(1:string db_name,
|
|
|
+ 2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1)
|
|
|
+ throws(1:MetaException o1)
|
|
|
+
|
|
|
+ // get the partitions matching the given partition filter
|
|
|
+ list<Partition> get_partitions_by_filter(1:string db_name 2:string tbl_name
|
|
|
+ 3:string filter, 4:i16 max_parts=-1)
|
|
|
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
|
|
|
|
|
|
// changes the partition to the new partition object. partition is identified from the part values
|
|
|
// in the new_part
|
|
|
+ // * See notes on DDL_TIME
|
|
|
void alter_partition(1:string db_name, 2:string tbl_name, 3:Partition new_part)
|
|
|
throws(1:InvalidOperationException o1, 2:MetaException o2)
|
|
|
|
|
|
@@ -228,8 +330,81 @@ service ThriftHiveMetastore extends fb303.FacebookService
|
|
|
// thrown.
|
|
|
string get_config_value(1:string name, 2:string defaultValue)
|
|
|
throws(1:ConfigValSecurityException o1)
|
|
|
+
|
|
|
+ // converts a partition name into a partition values array
|
|
|
+ list<string> partition_name_to_vals(1: string part_name)
|
|
|
+ throws(1: MetaException o1)
|
|
|
+ // converts a partition name into a partition specification (a mapping from
|
|
|
+ // the partition cols to the values)
|
|
|
+ map<string, string> partition_name_to_spec(1: string part_name)
|
|
|
+ throws(1: MetaException o1)
|
|
|
+
|
|
|
+ //index
|
|
|
+ Index add_index(1:Index new_index, 2: Table index_table)
|
|
|
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
|
|
|
+ void alter_index(1:string dbname, 2:string base_tbl_name, 3:string idx_name, 4:Index new_idx)
|
|
|
+ throws (1:InvalidOperationException o1, 2:MetaException o2)
|
|
|
+ bool drop_index_by_name(1:string db_name, 2:string tbl_name, 3:string index_name, 4:bool deleteData)
|
|
|
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
|
|
|
+ Index get_index_by_name(1:string db_name 2:string tbl_name, 3:string index_name)
|
|
|
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
|
|
|
+
|
|
|
+ list<Index> get_indexes(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
|
|
|
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
|
|
|
+ list<string> get_index_names(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
|
|
|
+ throws(1:MetaException o2)
|
|
|
+
|
|
|
+ //authorization privileges
|
|
|
+
|
|
|
+ bool create_role(1:Role role) throws(1:MetaException o1)
|
|
|
+ bool drop_role(1:string role_name) throws(1:MetaException o1)
|
|
|
+ list<string> get_role_names() throws(1:MetaException o1)
|
|
|
+ bool grant_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type,
|
|
|
+ 4:string grantor, 5:PrincipalType grantorType, 6:bool grant_option) throws(1:MetaException o1)
|
|
|
+ bool revoke_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type)
|
|
|
+ throws(1:MetaException o1)
|
|
|
+ list<Role> list_roles(1:string principal_name, 2:PrincipalType principal_type) throws(1:MetaException o1)
|
|
|
+
|
|
|
+ PrincipalPrivilegeSet get_privilege_set(1:HiveObjectRef hiveObject, 2:string user_name,
|
|
|
+ 3: list<string> group_names) throws(1:MetaException o1)
|
|
|
+ list<HiveObjectPrivilege> list_privileges(1:string principal_name, 2:PrincipalType principal_type,
|
|
|
+ 3: HiveObjectRef hiveObject) throws(1:MetaException o1)
|
|
|
+
|
|
|
+ bool grant_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
|
|
|
+ bool revoke_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
|
|
|
+
|
|
|
+ //Authentication (delegation token) interfaces
|
|
|
+
|
|
|
+ // get metastore server delegation token for use from the map/reduce tasks to authenticate
|
|
|
+ // to metastore server
|
|
|
+ string get_delegation_token(1:string renewer_kerberos_principal_name) throws (1:MetaException o1)
|
|
|
+
|
|
|
+ // get metastore server delegation token for use from the map/reduce tasks to authenticate
|
|
|
+ // to metastore server - this method takes an extra token signature string which is just
|
|
|
+ // an identifier to associate with the token - this will be used by the token selector code
|
|
|
+ // to pick the right token given the associated identifier.
|
|
|
+ string get_delegation_token_with_signature(1:string renewer_kerberos_principal_name,
|
|
|
+ 2:string token_signature) throws (1:MetaException o1)
|
|
|
+
|
|
|
+ // method to renew delegation token obtained from metastore server
|
|
|
+ i64 renew_delegation_token(1:string token_str_form) throws (1:MetaException o1)
|
|
|
+
|
|
|
+ // method to cancel delegation token obtained from metastore server
|
|
|
+ void cancel_delegation_token(1:string token_str_form) throws (1:MetaException o1)
|
|
|
}
|
|
|
|
|
|
+// * Note about the DDL_TIME: When creating or altering a table or a partition,
|
|
|
+// if the DDL_TIME is not set, the current time will be used.
|
|
|
+
|
|
|
+// For storing info about archived partitions in parameters
|
|
|
+
|
|
|
+// Whether the partition is archived
|
|
|
+const string IS_ARCHIVED = "is_archived",
|
|
|
+// The original location of the partition, before archiving. After archiving,
|
|
|
+// this directory will contain the archive. When the partition
|
|
|
+// is dropped, this directory will be deleted
|
|
|
+const string ORIGINAL_LOCATION = "original_location",
|
|
|
+
|
|
|
// these should be needed only for backward compatibility with filestore
|
|
|
const string META_TABLE_COLUMNS = "columns",
|
|
|
const string META_TABLE_COLUMN_TYPES = "columns.types",
|
|
|
@@ -243,5 +418,7 @@ const string META_TABLE_SERDE = "serde",
|
|
|
const string META_TABLE_PARTITION_COLUMNS = "partition_columns",
|
|
|
const string FILE_INPUT_FORMAT = "file.inputformat",
|
|
|
const string FILE_OUTPUT_FORMAT = "file.outputformat",
|
|
|
+const string META_TABLE_STORAGE = "storage_handler",
|
|
|
+
|
|
|
|
|
|
|