hdfs.thrift 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532
  1. /*
  2. * Licensed to Cloudera, Inc. under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. Cloudera, Inc. licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. /*
  19. * Thrift interface for HDFS.
  20. */
  21. /* Common types and interfaces */
  22. include 'common.thrift'
  23. /*
  24. * Namespaces for generated code. The idea is to keep code generated by
  25. * Thrift under a 'hadoop.api' namespace, so that a higher-level set of
  26. * functions and classes may be defined under 'hadoop'.
  27. */
  28. namespace cpp hadoop.api.hdfs
  29. namespace csharp Hadoop.API.HDFS
  30. namespace java org.apache.hadoop.thriftfs.api
  31. namespace perl Hadoop.API.HDFS
  32. namespace php hadoop_api_hdfs
  33. namespace py hadoop.api.hdfs
  34. namespace rb Hadoop.API.HDFS
  35. /* Values for 'type' argument to getDatanodeReport(). */
  36. enum DatanodeReportType {
  37. ALL_DATANODES = 1;
  38. LIVE_DATANODES = 2;
  39. DEAD_DATANODES = 3;
  40. }
  41. /* Values for DatanodeInfo.state */
  42. enum DatanodeState {
  43. NORMAL_STATE = 1;
  44. DECOMMISSION_INPROGRESS = 2;
  45. DECOMMISSIONED = 3;
  46. }
  47. /* Value for unknown Thrift port in DatanodeInfo */
  48. const i32 UNKNOWN_THRIFT_PORT = -1;
  49. /* Values for setQuota() parameters. */
  50. const i64 QUOTA_DONT_SET = -2
  51. /* Clear quota on this path. */
  52. const i64 QUOTA_RESET = -1
  53. /**
  54. * Information and state of a data node.
  55. *
  56. * Modelled after org.apache.hadoop.hdfs.protocol.DatanodeInfo
  57. */
  58. struct DatanodeInfo {
  59. /** HDFS name of the datanode (equals to <host>:<datanode port>) */
  60. 1: string name,
  61. /** Unique ID within a HDFS cluster */
  62. 2: string storageID,
  63. /** Host name of the Thrift server socket. */
  64. 3: string host,
  65. /** Port number of the Thrift server socket, or UNKNOWN_THRIFT_PORT
  66. if the Thrift port for this datanode is not known. */
  67. 4: i32 thriftPort,
  68. /** Port number of the Web UI */
  69. 10: i32 httpPort,
  70. /** Raw capacity of the data node (in bytes). */
  71. 5: i64 capacity,
  72. /** Space used by the data node (in bytes). */
  73. 6: i64 dfsUsed,
  74. /** Raw free space in the data node (in bytes). */
  75. 7: i64 remaining,
  76. /** Number of active connections to the data node. */
  77. 8: i32 xceiverCount,
  78. /** State of this data node. */
  79. 9: DatanodeState state,
  80. /** Number of seconds since last contact */
  81. 11: i64 millisSinceUpdate,
  82. }
  83. /**
  84. * Representation of a file block in HDFS
  85. *
  86. * Modelled after org.apache.hadoop.hdfs.protocol.LocatedBlock
  87. */
  88. struct Block {
  89. /** Block ID (unique among all blocks in a filesystem). */
  90. 1: i64 blockId,
  91. /** Path of the file which this block belongs to. */
  92. 2: string path,
  93. /** Length of this block. */
  94. 3: i64 numBytes,
  95. /** Generational stamp of this block. */
  96. 4: i64 genStamp,
  97. /** Offset of the first byte of the block relative to the start of the file */
  98. 6: i64 startOffset;
  99. /** List of data nodes with copies of this block. */
  100. 5: list<DatanodeInfo> nodes,
  101. }
  102. /**
  103. * Information about a path in HDFS.
  104. *
  105. * Modelled after org.apache.hadoop.fs.FileStatus
  106. */
  107. struct Stat {
  108. /** The path. */
  109. 1: string path,
  110. /**
  111. * True: The path represents a file.
  112. * False: The path represents a directory.
  113. */
  114. 2: bool isDir,
  115. /* Fields common to file and directory paths. */
  116. /** Access time (milliseconds since 1970-01-01 00:00 UTC). */
  117. 3: i64 atime,
  118. /** Modification time (milliseconds since 1970-01-01 00:00 UTC). */
  119. 4: i64 mtime,
  120. /** Access permissions */
  121. 5: i16 perms,
  122. /** Owner */
  123. 6: string owner,
  124. /** Group */
  125. 7: string group,
  126. /* Fields for file paths (will be zero for directory entries). */
  127. /** Length (in bytes). */
  128. 13: i64 length,
  129. /** Block size (in bytes). */
  130. 14: i64 blockSize,
  131. /** Replication factor. */
  132. 15: i16 replication
  133. }
  134. /**
  135. * Information about an entire subtree under a directory
  136. * Includes the information from org.apache.hadoop.fs.ContentSummary
  137. */
  138. struct ContentSummary {
  139. /* Fields for directory paths (will be zero for file entries). */
  140. /** Number of files in this directory */
  141. 1: i64 fileCount,
  142. /** Number of directories in this directory */
  143. 2: i64 directoryCount,
  144. /** Quota for this directory (number of files). */
  145. 3: i64 quota,
  146. /** Space consumed in disk (in bytes). */
  147. 4: i64 spaceConsumed,
  148. /** Quota consumed in disk (in bytes). */
  149. 5: i64 spaceQuota,
  150. /** The path */
  151. 6: string path,
  152. }
  153. struct UpgradeStatusReport {
  154. 1: i32 version
  155. 2: i16 percentComplete
  156. 3: bool finalized
  157. /** The informative text that is the same as is shown on the NN web UI */
  158. 4: string statusText
  159. }
  160. /**
  161. * Information that mirrors the "health report" information available on the
  162. * NameNode web UI
  163. */
  164. struct DFSHealthReport {
  165. 1: i64 bytesTotal
  166. 2: i64 bytesUsed
  167. 3: i64 bytesRemaining
  168. 4: i64 bytesNonDfs
  169. /** How many datanodes are considered live */
  170. 5: i32 numLiveDataNodes
  171. /** How many datanodes are considered dead */
  172. 6: i32 numDeadDataNodes
  173. /**
  174. * Status of the current running upgrade. If no upgrade
  175. * is running, this will be null.
  176. */
  177. 7: UpgradeStatusReport upgradeStatus
  178. /**
  179. * The http port that the NameNode is listening on for its web UI
  180. * - this isn't really health, but it's related and handy
  181. */
  182. 8: i32 httpPort
  183. }
  184. /** Quota-related error */
  185. exception QuotaException {
  186. /** Error message. */
  187. 1: string msg,
  188. /** Textual representation of the call stack. */
  189. 2: string stack
  190. }
  191. /**
  192. * Provides an interface to a Hadoop Namenode. It is basically a Thrift
  193. * translation of org.apache.hadoop.hdfs.protocol.ClientProtocol.
  194. */
  195. service Namenode extends common.HadoopServiceBase {
  196. /** Set permissions of an existing file or directory. */
  197. void chmod(10: common.RequestContext ctx,
  198. /** Path of the file or directory. */
  199. 1: string path,
  200. /** New permissions for the file or directory. */
  201. 2: i16 perms) throws (1: common.IOException err),
  202. /**
  203. * Set owner of a file or directory.
  204. *
  205. * If either parameter 'owner' or 'group' is set to null, that
  206. * parameter is left unchanged.
  207. *
  208. * Parameters 'owner' and 'group' cannot be both null.
  209. */
  210. void chown(10: common.RequestContext ctx,
  211. /** Path to the file or directory */
  212. 1: string path,
  213. /** New owner. */
  214. 2: string owner,
  215. /** New group. */
  216. 3: string group) throws (1: common.IOException err),
  217. /**
  218. * Return a list containing:
  219. * (index 0) The total storage capacity of the file system (in bytes).
  220. * (index 1) The total used space of the file system (in bytes).
  221. * (index 2) The available storage of the file system (in bytes).
  222. */
  223. list<i64> df(10: common.RequestContext ctx) throws (1: common.IOException err),
  224. /**
  225. * Enter safe mode.
  226. */
  227. void enterSafeMode(10: common.RequestContext ctx) throws (1: common.IOException err),
  228. /** Get a list of all blocks containing a region of a file */
  229. list<Block> getBlocks(10: common.RequestContext ctx,
  230. /** Path to the file. */
  231. 1: string path,
  232. /** Offset of the region. */
  233. 2: i64 offset,
  234. /** Length of the region */
  235. 3: i64 length) throws (1: common.IOException err),
  236. /** Get a report on the system's current data nodes.
  237. Note that ctx is currently ignored by the server. */
  238. list<DatanodeInfo> getDatanodeReport(10: common.RequestContext ctx,
  239. /**
  240. * Type of data nodes to return
  241. * information about.
  242. */
  243. 1: DatanodeReportType type)
  244. throws (1: common.IOException err),
  245. /** Get a health report of DFS. Note that ctx is ignored by the server. */
  246. DFSHealthReport getHealthReport(10: common.RequestContext ctx) throws (1: common.IOException err),
  247. /**
  248. * Get the preferred block size for the given file.
  249. *
  250. * The path must exist, or common.IOException is thrown.
  251. */
  252. i64 getPreferredBlockSize(10: common.RequestContext ctx,
  253. /** Path to the file. */
  254. 1: string path) throws (1: common.IOException err),
  255. /**
  256. * Returns whether HDFS is in safe mode or not.
  257. */
  258. bool isInSafeMode(10: common.RequestContext ctx) throws (1: common.IOException err),
  259. /**
  260. * Leave safe mode.
  261. */
  262. void leaveSafeMode(10: common.RequestContext ctx) throws (1: common.IOException err),
  263. /** Get a listing of the indicated directory. */
  264. list<Stat> ls(10: common.RequestContext ctx,
  265. /** Path to the directory. */
  266. 1: string path) throws (1: common.IOException err),
  267. /**
  268. * Create a directory (or hierarchy of directories).
  269. *
  270. * Returns false if directory did not exist and could not be created,
  271. * true otherwise.
  272. */
  273. bool mkdirhier(10: common.RequestContext ctx,
  274. /** Path to the directory. */
  275. 1: string path,
  276. /** Access permissions of the directory. */
  277. 2: i16 perms) throws (1: common.IOException err),
  278. /** Tells the name node to reread the hosts and exclude files. */
  279. void refreshNodes(10: common.RequestContext ctx) throws (1: common.IOException err),
  280. /**
  281. * Rename an item in the file system namespace.
  282. *
  283. * Returns true if successful, or
  284. * false if the old name does not exist or if the new name already
  285. * belongs to the namespace.
  286. */
  287. bool rename(10: common.RequestContext ctx,
  288. /** Path to existing file or directory. */
  289. 1: string path,
  290. /** New path. */
  291. 2: string newPath) throws (1: common.IOException err),
  292. /** Report corrupted blocks. */
  293. void reportBadBlocks(10: common.RequestContext ctx,
  294. /** List of corrupted blocks. */
  295. 1: list<Block> blocks) throws (1: common.IOException err),
  296. /**
  297. * Get information about a path in HDFS.
  298. *
  299. * Return value will be nul if path does not exist.
  300. */
  301. Stat stat(10: common.RequestContext ctx,
  302. /** Path of the file or directory. */
  303. 1: string path) throws (1: common.IOException err),
  304. /**
  305. * Get the summary of a directory's contents.
  306. *
  307. * Note that this has runtime linear in the total number of nodes
  308. * in the directory tree - this can be expensive for directories
  309. * near the top of a big HDFS. Use with care.
  310. */
  311. ContentSummary getContentSummary(10: common.RequestContext ctx,
  312. 1: string Path) throws (1: common.IOException err),
  313. /**
  314. * Get ContentSummary objects for multiple directories simultaneously. The same warnings
  315. * apply as for getContentSummary(...) above.
  316. */
  317. list<ContentSummary> multiGetContentSummary(10: common.RequestContext ctx,
  318. 1: list<string> paths) throws (1: common.IOException err),
  319. /**
  320. * Set the quota for a directory.
  321. *
  322. * Quota parameters may have three types of values:
  323. *
  324. * (1) 0 or more: Quota will be set to that value.
  325. * (2) QUOTA_DONT_SET: Quota will not be changed,
  326. * (3) QUOTA_RESET: Quota will be reset.
  327. *
  328. * Any other value is a runtime error.
  329. */
  330. void setQuota(10: common.RequestContext ctx,
  331. /** Path of the directory. */
  332. 1: string path,
  333. /** Limit on the number of names in the directory. */
  334. 2: i64 namespaceQuota,
  335. /**
  336. * Limit on disk space occupied by all the files in the
  337. * directory.
  338. */
  339. 3: i64 diskspaceQuota) throws (1: common.IOException err),
  340. /**
  341. * Set replication factor for an existing file.
  342. *
  343. * This call just updates the value of the replication factor. The actual
  344. * block replication is not expected to be performed during this method call.
  345. * The blocks will be populated or removed in the background as the result of
  346. * the routine block maintenance procedures.
  347. *
  348. * Returns true if successful, false if file does not exist or is a
  349. * directory.
  350. */
  351. bool setReplication(10: common.RequestContext ctx,
  352. /** Path of the file. */
  353. 1: string path,
  354. /** New replication factor. */
  355. 2: i16 replication) throws (1: common.IOException err),
  356. /**
  357. * Delete a file or directory from the file system.
  358. *
  359. * Any blocks belonging to the deleted files will be garbage-collected.
  360. */
  361. bool unlink(10: common.RequestContext ctx,
  362. /** Path of the file or directory. */
  363. 1: string path,
  364. /** Delete a non-empty directory recursively. */
  365. 2: bool recursive) throws (1: common.IOException err),
  366. /**
  367. * Sets the modification and access time of a file or directory.
  368. *
  369. * Setting *one single time paramater* to -1 means that time parameter
  370. * must not be set by this call.
  371. *
  372. * Setting *both time parameters* to -1 means both of them must be set to
  373. * the current time.
  374. */
  375. void utime(10: common.RequestContext ctx,
  376. /** Path of the file or directory. */
  377. 1: string path,
  378. /** Access time in milliseconds since 1970-01-01 00:00 UTC */
  379. 2: i64 atime,
  380. /** Modification time in milliseconds since 1970-01-01 00:00 UTC */
  381. 3: i64 mtime) throws (1: common.IOException err),
  382. /*
  383. * The following methods are meant to be called by datanodes to advertise
  384. * themselves to the namenode.
  385. */
  386. /**
  387. * Inform the namenode that a datanode process has started.
  388. */
  389. void datanodeUp(/** <host name>:<port number> of the datanode */
  390. 1: string name,
  391. /** the storage id of the datanode */
  392. 2: string storage,
  393. /** Thrift port of the datanode */
  394. 3: i32 thriftPort),
  395. /**
  396. * Inform the namenode that a datanode process has stopped.
  397. */
  398. void datanodeDown(/** <host name>:<port number> of the datanode */
  399. 1: string name,
  400. /** the storage id of the datanode */
  401. 2: string storage,
  402. /** Thrift port of the datanode */
  403. 3: i32 thriftPort),
  404. }
  405. /** Encapsulates a block data transfer with its CRC */
  406. struct BlockData {
  407. /** CRC32 of the data being transfered */
  408. 1: i32 crc,
  409. /** Length of the data being transfered */
  410. 2: i32 length,
  411. /** The data itsef */
  412. 3: binary data
  413. }
  414. /**
  415. * Provides an interface to data nodes, so that clients may read and write
  416. * data blocks.
  417. */
  418. service Datanode {
  419. /**
  420. * Read bytes from a block.
  421. *
  422. * Only 2^31 - 1 bytes may be read on a single call to this method.
  423. */
  424. BlockData readBlock(10: common.RequestContext ctx,
  425. /** Block to be read from. */
  426. 1: Block block,
  427. /** Offset within the block where read must start from. */
  428. 2: i64 offset,
  429. /** Number of bytes to read. */
  430. 3: i32 length) throws (1:common.IOException err)
  431. }