hive_metastore.thrift 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424
  1. #!/usr/local/bin/thrift -java
  2. // Licensed to Cloudera, Inc. under one
  3. // or more contributor license agreements. See the NOTICE file
  4. // distributed with this work for additional information
  5. // regarding copyright ownership. Cloudera, Inc. licenses this file
  6. // to you under the Apache License, Version 2.0 (the
  7. // "License"); you may not use this file except in compliance
  8. // with the License. You may obtain a copy of the License at
  9. //
  10. // http://www.apache.org/licenses/LICENSE-2.0
  11. //
  12. // Unless required by applicable law or agreed to in writing, software
  13. // distributed under the License is distributed on an "AS IS" BASIS,
  14. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. // See the License for the specific language governing permissions and
  16. // limitations under the License.
  17. #
  18. # Thrift Service that the MetaStore is built on
  19. #
  20. include "fb303.thrift"
  21. namespace java org.apache.hadoop.hive.metastore.api
  22. namespace php metastore
  23. namespace cpp Apache.Hadoop.Hive
  24. const string DDL_TIME = "transient_lastDdlTime"
  25. struct Version {
  26. 1: string version,
  27. 2: string comments
  28. }
  29. struct FieldSchema {
  30. 1: string name, // name of the field
  31. 2: string type, // type of the field. primitive types defined above, specify list<TYPE_NAME>, map<TYPE_NAME, TYPE_NAME> for lists & maps
  32. 3: string comment
  33. }
  34. struct Type {
  35. 1: string name, // one of the types in PrimitiveTypes or CollectionTypes or User defined types
  36. 2: optional string type1, // object type if the name is 'list' (LIST_TYPE), key type if the name is 'map' (MAP_TYPE)
  37. 3: optional string type2, // val type if the name is 'map' (MAP_TYPE)
  38. 4: optional list<FieldSchema> fields // if the name is one of the user defined types
  39. }
  40. enum HiveObjectType {
  41. GLOBAL = 1,
  42. DATABASE = 2,
  43. TABLE = 3,
  44. PARTITION = 4,
  45. COLUMN = 5,
  46. }
  47. enum PrincipalType {
  48. USER = 1,
  49. ROLE = 2,
  50. GROUP = 3,
  51. }
  52. struct HiveObjectRef{
  53. 1: HiveObjectType objectType,
  54. 2: string dbName,
  55. 3: string objectName,
  56. 4: list<string> partValues,
  57. 5: string columnName,
  58. }
  59. struct PrivilegeGrantInfo {
  60. 1: string privilege,
  61. 2: i32 createTime,
  62. 3: string grantor,
  63. 4: PrincipalType grantorType,
  64. 5: bool grantOption,
  65. }
  66. struct HiveObjectPrivilege {
  67. 1: HiveObjectRef hiveObject,
  68. 2: string principalName,
  69. 3: PrincipalType principalType,
  70. 4: PrivilegeGrantInfo grantInfo,
  71. }
  72. struct PrivilegeBag {
  73. 1: list<HiveObjectPrivilege> privileges,
  74. }
  75. struct PrincipalPrivilegeSet {
  76. 1: map<string, list<PrivilegeGrantInfo>> userPrivileges, // user name -> privilege grant info
  77. 2: map<string, list<PrivilegeGrantInfo>> groupPrivileges, // group name -> privilege grant info
  78. 3: map<string, list<PrivilegeGrantInfo>> rolePrivileges, //role name -> privilege grant info
  79. }
  80. struct Role {
  81. 1: string roleName,
  82. 2: i32 createTime,
  83. 3: string ownerName,
  84. }
  85. // namespace for tables
  86. struct Database {
  87. 1: string name,
  88. 2: string description,
  89. 3: string locationUri,
  90. 4: map<string, string> parameters, // properties associated with the database
  91. 5: optional PrincipalPrivilegeSet privileges
  92. }
  93. // This object holds the information needed by SerDes
  94. struct SerDeInfo {
  95. 1: string name, // name of the serde, table name by default
  96. 2: string serializationLib, // usually the class that implements the extractor & loader
  97. 3: map<string, string> parameters // initialization parameters
  98. }
  99. // sort order of a column (column name along with asc(1)/desc(0))
  100. struct Order {
  101. 1: string col, // sort column name
  102. 2: i32 order // asc(1) or desc(0)
  103. }
  104. // this object holds all the information about physical storage of the data belonging to a table
  105. struct StorageDescriptor {
  106. 1: list<FieldSchema> cols, // required (refer to types defined above)
  107. 2: string location, // defaults to <warehouse loc>/<db loc>/tablename
  108. 3: string inputFormat, // SequenceFileInputFormat (binary) or TextInputFormat` or custom format
  109. 4: string outputFormat, // SequenceFileOutputFormat (binary) or IgnoreKeyTextOutputFormat or custom format
  110. 5: bool compressed, // compressed or not
  111. 6: i32 numBuckets, // this must be specified if there are any dimension columns
  112. 7: SerDeInfo serdeInfo, // serialization and deserialization information
  113. 8: list<string> bucketCols, // reducer grouping columns and clustering columns and bucketing columns`
  114. 9: list<Order> sortCols, // sort order of the data in each bucket
  115. 10: map<string, string> parameters // any user supplied key value hash
  116. }
  117. // table information
  118. struct Table {
  119. 1: string tableName, // name of the table
  120. 2: string dbName, // database name ('default')
  121. 3: string owner, // owner of this table
  122. 4: i32 createTime, // creation time of the table
  123. 5: i32 lastAccessTime, // last access time (usually this will be filled from HDFS and shouldn't be relied on)
  124. 6: i32 retention, // retention time
  125. 7: StorageDescriptor sd, // storage descriptor of the table
  126. 8: list<FieldSchema> partitionKeys, // partition keys of the table. only primitive types are supported
  127. 9: map<string, string> parameters, // to store comments or any other user level parameters
  128. 10: string viewOriginalText, // original view text, null for non-view
  129. 11: string viewExpandedText, // expanded view text, null for non-view
  130. 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
  131. 13: optional PrincipalPrivilegeSet privileges,
  132. }
  133. struct Partition {
  134. 1: list<string> values // string value is converted to appropriate partition key type
  135. 2: string dbName,
  136. 3: string tableName,
  137. 4: i32 createTime,
  138. 5: i32 lastAccessTime,
  139. 6: StorageDescriptor sd,
  140. 7: map<string, string> parameters,
  141. 8: optional PrincipalPrivilegeSet privileges
  142. }
  143. struct Index {
  144. 1: string indexName, // unique with in the whole database namespace
  145. 2: string indexHandlerClass, // reserved
  146. 3: string dbName,
  147. 4: string origTableName,
  148. 5: i32 createTime,
  149. 6: i32 lastAccessTime,
  150. 7: string indexTableName,
  151. 8: StorageDescriptor sd,
  152. 9: map<string, string> parameters,
  153. 10: bool deferredRebuild
  154. }
  155. // schema of the table/query results etc.
  156. struct Schema {
  157. // column names, types, comments
  158. 1: list<FieldSchema> fieldSchemas, // delimiters etc
  159. 2: map<string, string> properties
  160. }
  161. exception MetaException {
  162. 1: string message
  163. }
  164. exception UnknownTableException {
  165. 1: string message
  166. }
  167. exception UnknownDBException {
  168. 1: string message
  169. }
  170. exception AlreadyExistsException {
  171. 1: string message
  172. }
  173. exception InvalidObjectException {
  174. 1: string message
  175. }
  176. exception NoSuchObjectException {
  177. 1: string message
  178. }
  179. exception IndexAlreadyExistsException {
  180. 1: string message
  181. }
  182. exception InvalidOperationException {
  183. 1: string message
  184. }
  185. exception ConfigValSecurityException {
  186. 1: string message
  187. }
  188. /**
  189. * This interface is live.
  190. */
  191. service ThriftHiveMetastore extends fb303.FacebookService
  192. {
  193. void create_database(1:Database database) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
  194. Database get_database(1:string name) throws(1:NoSuchObjectException o1, 2:MetaException o2)
  195. void drop_database(1:string name, 2:bool deleteData) throws(1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3)
  196. list<string> get_databases(1:string pattern) throws(1:MetaException o1)
  197. list<string> get_all_databases() throws(1:MetaException o1)
  198. void alter_database(1:string dbname, 2:Database db) throws(1:MetaException o1, 2:NoSuchObjectException o2)
  199. // returns the type with given name (make seperate calls for the dependent types if needed)
  200. Type get_type(1:string name) throws(1:MetaException o1, 2:NoSuchObjectException o2)
  201. bool create_type(1:Type type) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
  202. bool drop_type(1:string type) throws(1:MetaException o1, 2:NoSuchObjectException o2)
  203. map<string, Type> get_type_all(1:string name)
  204. throws(1:MetaException o2)
  205. // Gets a list of FieldSchemas describing the columns of a particular table
  206. list<FieldSchema> get_fields(1: string db_name, 2: string table_name) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3),
  207. // Gets a list of FieldSchemas describing both the columns and the partition keys of a particular table
  208. list<FieldSchema> get_schema(1: string db_name, 2: string table_name) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3)
  209. // create a Hive table. Following fields must be set
  210. // tableName
  211. // database (only 'default' for now until Hive QL supports databases)
  212. // owner (not needed, but good to have for tracking purposes)
  213. // sd.cols (list of field schemas)
  214. // sd.inputFormat (SequenceFileInputFormat (binary like falcon tables or u_full) or TextInputFormat)
  215. // sd.outputFormat (SequenceFileInputFormat (binary) or TextInputFormat)
  216. // sd.serdeInfo.serializationLib (SerDe class name eg org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe
  217. // * See notes on DDL_TIME
  218. void create_table(1:Table tbl) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:NoSuchObjectException o4)
  219. // drops the table and all the partitions associated with it if the table has partitions
  220. // delete data (including partitions) if deleteData is set to true
  221. void drop_table(1:string dbname, 2:string name, 3:bool deleteData)
  222. throws(1:NoSuchObjectException o1, 2:MetaException o3)
  223. list<string> get_tables(1: string db_name, 2: string pattern) throws (1: MetaException o1)
  224. list<string> get_all_tables(1: string db_name) throws (1: MetaException o1)
  225. Table get_table(1:string dbname, 2:string tbl_name)
  226. throws (1:MetaException o1, 2:NoSuchObjectException o2)
  227. // alter table applies to only future partitions not for existing partitions
  228. // * See notes on DDL_TIME
  229. void alter_table(1:string dbname, 2:string tbl_name, 3:Table new_tbl)
  230. throws (1:InvalidOperationException o1, 2:MetaException o2)
  231. // the following applies to only tables that have partitions
  232. // * See notes on DDL_TIME
  233. Partition add_partition(1:Partition new_part)
  234. throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
  235. Partition append_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
  236. throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
  237. Partition append_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name)
  238. throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
  239. bool drop_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:bool deleteData)
  240. throws(1:NoSuchObjectException o1, 2:MetaException o2)
  241. bool drop_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name, 4:bool deleteData)
  242. throws(1:NoSuchObjectException o1, 2:MetaException o2)
  243. Partition get_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
  244. throws(1:MetaException o1, 2:NoSuchObjectException o2)
  245. Partition get_partition_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals,
  246. 4: string user_name, 5: list<string> group_names) throws(1:MetaException o1, 2:NoSuchObjectException o2)
  247. Partition get_partition_by_name(1:string db_name 2:string tbl_name, 3:string part_name)
  248. throws(1:MetaException o1, 2:NoSuchObjectException o2)
  249. // returns all the partitions for this table in reverse chronological order.
  250. // If max parts is given then it will return only that many.
  251. list<Partition> get_partitions(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
  252. throws(1:NoSuchObjectException o1, 2:MetaException o2)
  253. list<Partition> get_partitions_with_auth(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1,
  254. 4: string user_name, 5: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
  255. list<string> get_partition_names(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
  256. throws(1:MetaException o2)
  257. // get_partition*_ps methods allow filtering by a partial partition specification,
  258. // as needed for dynamic partitions. The values that are not restricted should
  259. // be empty strings. Nulls were considered (instead of "") but caused errors in
  260. // generated Python code. The size of part_vals may be smaller than the
  261. // number of partition columns - the unspecified values are considered the same
  262. // as "".
  263. list<Partition> get_partitions_ps(1:string db_name 2:string tbl_name
  264. 3:list<string> part_vals, 4:i16 max_parts=-1)
  265. throws(1:MetaException o1)
  266. list<Partition> get_partitions_ps_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1,
  267. 5: string user_name, 6: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
  268. list<string> get_partition_names_ps(1:string db_name,
  269. 2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1)
  270. throws(1:MetaException o1)
  271. // get the partitions matching the given partition filter
  272. list<Partition> get_partitions_by_filter(1:string db_name 2:string tbl_name
  273. 3:string filter, 4:i16 max_parts=-1)
  274. throws(1:MetaException o1, 2:NoSuchObjectException o2)
  275. // changes the partition to the new partition object. partition is identified from the part values
  276. // in the new_part
  277. // * See notes on DDL_TIME
  278. void alter_partition(1:string db_name, 2:string tbl_name, 3:Partition new_part)
  279. throws(1:InvalidOperationException o1, 2:MetaException o2)
  280. // gets the value of the configuration key in the metastore server. returns
  281. // defaultValue if the key does not exist. if the configuration key does not
  282. // begin with "hive", "mapred", or "hdfs", a ConfigValSecurityException is
  283. // thrown.
  284. string get_config_value(1:string name, 2:string defaultValue)
  285. throws(1:ConfigValSecurityException o1)
  286. // converts a partition name into a partition values array
  287. list<string> partition_name_to_vals(1: string part_name)
  288. throws(1: MetaException o1)
  289. // converts a partition name into a partition specification (a mapping from
  290. // the partition cols to the values)
  291. map<string, string> partition_name_to_spec(1: string part_name)
  292. throws(1: MetaException o1)
  293. //index
  294. Index add_index(1:Index new_index, 2: Table index_table)
  295. throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
  296. void alter_index(1:string dbname, 2:string base_tbl_name, 3:string idx_name, 4:Index new_idx)
  297. throws (1:InvalidOperationException o1, 2:MetaException o2)
  298. bool drop_index_by_name(1:string db_name, 2:string tbl_name, 3:string index_name, 4:bool deleteData)
  299. throws(1:NoSuchObjectException o1, 2:MetaException o2)
  300. Index get_index_by_name(1:string db_name 2:string tbl_name, 3:string index_name)
  301. throws(1:MetaException o1, 2:NoSuchObjectException o2)
  302. list<Index> get_indexes(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
  303. throws(1:NoSuchObjectException o1, 2:MetaException o2)
  304. list<string> get_index_names(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
  305. throws(1:MetaException o2)
  306. //authorization privileges
  307. bool create_role(1:Role role) throws(1:MetaException o1)
  308. bool drop_role(1:string role_name) throws(1:MetaException o1)
  309. list<string> get_role_names() throws(1:MetaException o1)
  310. bool grant_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type,
  311. 4:string grantor, 5:PrincipalType grantorType, 6:bool grant_option) throws(1:MetaException o1)
  312. bool revoke_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type)
  313. throws(1:MetaException o1)
  314. list<Role> list_roles(1:string principal_name, 2:PrincipalType principal_type) throws(1:MetaException o1)
  315. PrincipalPrivilegeSet get_privilege_set(1:HiveObjectRef hiveObject, 2:string user_name,
  316. 3: list<string> group_names) throws(1:MetaException o1)
  317. list<HiveObjectPrivilege> list_privileges(1:string principal_name, 2:PrincipalType principal_type,
  318. 3: HiveObjectRef hiveObject) throws(1:MetaException o1)
  319. bool grant_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
  320. bool revoke_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
  321. //Authentication (delegation token) interfaces
  322. // get metastore server delegation token for use from the map/reduce tasks to authenticate
  323. // to metastore server
  324. string get_delegation_token(1:string renewer_kerberos_principal_name) throws (1:MetaException o1)
  325. // get metastore server delegation token for use from the map/reduce tasks to authenticate
  326. // to metastore server - this method takes an extra token signature string which is just
  327. // an identifier to associate with the token - this will be used by the token selector code
  328. // to pick the right token given the associated identifier.
  329. string get_delegation_token_with_signature(1:string renewer_kerberos_principal_name,
  330. 2:string token_signature) throws (1:MetaException o1)
  331. // method to renew delegation token obtained from metastore server
  332. i64 renew_delegation_token(1:string token_str_form) throws (1:MetaException o1)
  333. // method to cancel delegation token obtained from metastore server
  334. void cancel_delegation_token(1:string token_str_form) throws (1:MetaException o1)
  335. }
  336. // * Note about the DDL_TIME: When creating or altering a table or a partition,
  337. // if the DDL_TIME is not set, the current time will be used.
  338. // For storing info about archived partitions in parameters
  339. // Whether the partition is archived
  340. const string IS_ARCHIVED = "is_archived",
  341. // The original location of the partition, before archiving. After archiving,
  342. // this directory will contain the archive. When the partition
  343. // is dropped, this directory will be deleted
  344. const string ORIGINAL_LOCATION = "original_location",
  345. // these should be needed only for backward compatibility with filestore
  346. const string META_TABLE_COLUMNS = "columns",
  347. const string META_TABLE_COLUMN_TYPES = "columns.types",
  348. const string BUCKET_FIELD_NAME = "bucket_field_name",
  349. const string BUCKET_COUNT = "bucket_count",
  350. const string FIELD_TO_DIMENSION = "field_to_dimension",
  351. const string META_TABLE_NAME = "name",
  352. const string META_TABLE_DB = "db",
  353. const string META_TABLE_LOCATION = "location",
  354. const string META_TABLE_SERDE = "serde",
  355. const string META_TABLE_PARTITION_COLUMNS = "partition_columns",
  356. const string FILE_INPUT_FORMAT = "file.inputformat",
  357. const string FILE_OUTPUT_FORMAT = "file.outputformat",
  358. const string META_TABLE_STORAGE = "storage_handler",