hdfs.thrift 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522
  1. /*
  2. * Licensed to Cloudera, Inc. under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. Cloudera, Inc. licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. /*
  19. * Thrift interface for HDFS.
  20. */
  21. /* Common types and interfaces */
  22. include 'common.thrift'
  23. /*
  24. * Namespaces for generated code. The idea is to keep code generated by
  25. * Thrift under a 'hadoop.api' namespace, so that a higher-level set of
  26. * functions and classes may be defined under 'hadoop'.
  27. */
  28. namespace cpp hadoop.api.hdfs
  29. namespace csharp Hadoop.API.HDFS
  30. namespace java org.apache.hadoop.thriftfs.api
  31. namespace perl Hadoop.API.HDFS
  32. namespace php hadoop_api_hdfs
  33. namespace py hadoop.api.hdfs
  34. namespace rb Hadoop.API.HDFS
  35. /* Values for 'type' argument to getDatanodeReport(). */
  36. enum DatanodeReportType {
  37. ALL_DATANODES = 1;
  38. LIVE_DATANODES = 2;
  39. DEAD_DATANODES = 3;
  40. }
  41. /* Values for DatanodeInfo.state */
  42. enum DatanodeState {
  43. NORMAL_STATE = 1;
  44. DECOMMISSION_INPROGRESS = 2;
  45. DECOMMISSIONED = 3;
  46. }
  47. /* Value for unknown Thrift port in DatanodeInfo */
  48. const i32 UNKNOWN_THRIFT_PORT = -1;
  49. /* Values for setQuota() parameters. */
  50. const i64 QUOTA_DONT_SET = -2
  51. /* Clear quota on this path. */
  52. const i64 QUOTA_RESET = -1
  53. /**
  54. * Information and state of a data node.
  55. *
  56. * Modelled after org.apache.hadoop.hdfs.protocol.DatanodeInfo
  57. */
  58. struct DatanodeInfo {
  59. /** HDFS name of the datanode (equals to <host>:<datanode port>) */
  60. 1: string name,
  61. /** Unique ID within a HDFS cluster */
  62. 2: string storageID,
  63. /** Host name of the Thrift server socket. */
  64. 3: string host,
  65. /** Port number of the Thrift server socket, or UNKNOWN_THRIFT_PORT
  66. if the Thrift port for this datanode is not known. */
  67. 4: i32 thriftPort,
  68. /** Port number of the Web UI */
  69. 10: i32 httpPort,
  70. /** Raw capacity of the data node (in bytes). */
  71. 5: i64 capacity,
  72. /** Space used by the data node (in bytes). */
  73. 6: i64 dfsUsed,
  74. /** Raw free space in the data node (in bytes). */
  75. 7: i64 remaining,
  76. /** Number of active connections to the data node. */
  77. 8: i32 xceiverCount,
  78. /** State of this data node. */
  79. 9: DatanodeState state,
  80. /** Number of seconds since last contact */
  81. 11: i64 millisSinceUpdate,
  82. }
  83. /**
  84. * Representation of a file block in HDFS
  85. *
  86. * Modelled after org.apache.hadoop.hdfs.protocol.LocatedBlock
  87. */
  88. struct Block {
  89. /** Block ID (unique among all blocks in a filesystem). */
  90. 1: i64 blockId,
  91. /** Path of the file which this block belongs to. */
  92. 2: string path,
  93. /** Length of this block. */
  94. 3: i64 numBytes,
  95. /** Generational stamp of this block. */
  96. 4: i64 genStamp,
  97. /** Offset of the first byte of the block relative to the start of the file */
  98. 6: i64 startOffset;
  99. /** List of data nodes with copies of this block. */
  100. 5: list<DatanodeInfo> nodes,
  101. }
  102. /**
  103. * Information about a path in HDFS.
  104. *
  105. * Modelled after org.apache.hadoop.fs.FileStatus
  106. */
  107. struct Stat {
  108. /** The path. */
  109. 1: string path,
  110. /**
  111. * True: The path represents a file.
  112. * False: The path represents a directory.
  113. */
  114. 2: bool isDir,
  115. /* Fields common to file and directory paths. */
  116. /** Access time (milliseconds since 1970-01-01 00:00 UTC). */
  117. 3: i64 atime,
  118. /** Modification time (milliseconds since 1970-01-01 00:00 UTC). */
  119. 4: i64 mtime,
  120. /** Access permissions */
  121. 5: i16 perms,
  122. /** Owner */
  123. 6: string owner,
  124. /** Group */
  125. 7: string group,
  126. /* Fields for file paths (will be zero for directory entries). */
  127. /** Length (in bytes). */
  128. 13: i64 length,
  129. /** Block size (in bytes). */
  130. 14: i64 blockSize,
  131. /** Replication factor. */
  132. 15: i16 replication
  133. }
  134. /**
  135. * Information about an entire subtree under a directory
  136. * Includes the information from org.apache.hadoop.fs.ContentSummary
  137. */
  138. struct ContentSummary {
  139. /* Fields for directory paths (will be zero for file entries). */
  140. /** Number of files in this directory */
  141. 1: i64 fileCount,
  142. /** Number of directories in this directory */
  143. 2: i64 directoryCount,
  144. /** Quota for this directory (number of files). */
  145. 3: i64 quota,
  146. /** Space consumed in disk (in bytes). */
  147. 4: i64 spaceConsumed,
  148. /** Quota consumed in disk (in bytes). */
  149. 5: i64 spaceQuota,
  150. }
  151. struct UpgradeStatusReport {
  152. 1: i32 version
  153. 2: i16 percentComplete
  154. 3: bool finalized
  155. /** The informative text that is the same as is shown on the NN web UI */
  156. 4: string statusText
  157. }
  158. /**
  159. * Information that mirrors the "health report" information available on the
  160. * NameNode web UI
  161. */
  162. struct DFSHealthReport {
  163. 1: i64 bytesTotal
  164. 2: i64 bytesUsed
  165. 3: i64 bytesRemaining
  166. 4: i64 bytesNonDfs
  167. /** How many datanodes are considered live */
  168. 5: i32 numLiveDataNodes
  169. /** How many datanodes are considered dead */
  170. 6: i32 numDeadDataNodes
  171. /**
  172. * Status of the current running upgrade. If no upgrade
  173. * is running, this will be null.
  174. */
  175. 7: UpgradeStatusReport upgradeStatus
  176. /**
  177. * The http port that the NameNode is listening on for its web UI
  178. * - this isn't really health, but it's related and handy
  179. */
  180. 8: i32 httpPort
  181. }
  182. /** Quota-related error */
  183. exception QuotaException {
  184. /** Error message. */
  185. 1: string msg,
  186. /** Textual representation of the call stack. */
  187. 2: string stack
  188. }
  189. /**
  190. * Provides an interface to a Hadoop Namenode. It is basically a Thrift
  191. * translation of org.apache.hadoop.hdfs.protocol.ClientProtocol.
  192. */
  193. service Namenode extends common.HadoopServiceBase {
  194. /** Set permissions of an existing file or directory. */
  195. void chmod(10: common.RequestContext ctx,
  196. /** Path of the file or directory. */
  197. 1: string path,
  198. /** New permissions for the file or directory. */
  199. 2: i16 perms) throws (1: common.IOException err),
  200. /**
  201. * Set owner of a file or directory.
  202. *
  203. * If either parameter 'owner' or 'group' is set to null, that
  204. * parameter is left unchanged.
  205. *
  206. * Parameters 'owner' and 'group' cannot be both null.
  207. */
  208. void chown(10: common.RequestContext ctx,
  209. /** Path to the file or directory */
  210. 1: string path,
  211. /** New owner. */
  212. 2: string owner,
  213. /** New group. */
  214. 3: string group) throws (1: common.IOException err),
  215. /**
  216. * Return a list containing:
  217. * (index 0) The total storage capacity of the file system (in bytes).
  218. * (index 1) The total used space of the file system (in bytes).
  219. * (index 2) The available storage of the file system (in bytes).
  220. */
  221. list<i64> df(10: common.RequestContext ctx) throws (1: common.IOException err),
  222. /**
  223. * Enter safe mode.
  224. */
  225. void enterSafeMode(10: common.RequestContext ctx) throws (1: common.IOException err),
  226. /** Get a list of all blocks containing a region of a file */
  227. list<Block> getBlocks(10: common.RequestContext ctx,
  228. /** Path to the file. */
  229. 1: string path,
  230. /** Offset of the region. */
  231. 2: i64 offset,
  232. /** Length of the region */
  233. 3: i64 length) throws (1: common.IOException err),
  234. /** Get a report on the system's current data nodes.
  235. Note that ctx is currently ignored by the server. */
  236. list<DatanodeInfo> getDatanodeReport(10: common.RequestContext ctx,
  237. /**
  238. * Type of data nodes to return
  239. * information about.
  240. */
  241. 1: DatanodeReportType type)
  242. throws (1: common.IOException err),
  243. /** Get a health report of DFS. Note that ctx is ignored by the server. */
  244. DFSHealthReport getHealthReport(10: common.RequestContext ctx) throws (1: common.IOException err),
  245. /**
  246. * Get the preferred block size for the given file.
  247. *
  248. * The path must exist, or common.IOException is thrown.
  249. */
  250. i64 getPreferredBlockSize(10: common.RequestContext ctx,
  251. /** Path to the file. */
  252. 1: string path) throws (1: common.IOException err),
  253. /**
  254. * Returns whether HDFS is in safe mode or not.
  255. */
  256. bool isInSafeMode(10: common.RequestContext ctx) throws (1: common.IOException err),
  257. /**
  258. * Leave safe mode.
  259. */
  260. void leaveSafeMode(10: common.RequestContext ctx) throws (1: common.IOException err),
  261. /** Get a listing of the indicated directory. */
  262. list<Stat> ls(10: common.RequestContext ctx,
  263. /** Path to the directory. */
  264. 1: string path) throws (1: common.IOException err),
  265. /**
  266. * Create a directory (or hierarchy of directories).
  267. *
  268. * Returns false if directory did not exist and could not be created,
  269. * true otherwise.
  270. */
  271. bool mkdirhier(10: common.RequestContext ctx,
  272. /** Path to the directory. */
  273. 1: string path,
  274. /** Access permissions of the directory. */
  275. 2: i16 perms) throws (1: common.IOException err),
  276. /** Tells the name node to reread the hosts and exclude files. */
  277. void refreshNodes(10: common.RequestContext ctx) throws (1: common.IOException err),
  278. /**
  279. * Rename an item in the file system namespace.
  280. *
  281. * Returns true if successful, or
  282. * false if the old name does not exist or if the new name already
  283. * belongs to the namespace.
  284. */
  285. bool rename(10: common.RequestContext ctx,
  286. /** Path to existing file or directory. */
  287. 1: string path,
  288. /** New path. */
  289. 2: string newPath) throws (1: common.IOException err),
  290. /** Report corrupted blocks. */
  291. void reportBadBlocks(10: common.RequestContext ctx,
  292. /** List of corrupted blocks. */
  293. 1: list<Block> blocks) throws (1: common.IOException err),
  294. /**
  295. * Get information about a path in HDFS.
  296. *
  297. * Return value will be nul if path does not exist.
  298. */
  299. Stat stat(10: common.RequestContext ctx,
  300. /** Path of the file or directory. */
  301. 1: string path) throws (1: common.IOException err),
  302. /**
  303. * Get the summary of a directory's contents.
  304. *
  305. * Note that this has runtime linear in the total number of nodes
  306. * in the directory tree - this can be expensive for directories
  307. * near the top of a big HDFS. Use with care.
  308. */
  309. ContentSummary getContentSummary(10: common.RequestContext ctx,
  310. 1: string Path) throws (1: common.IOException err),
  311. /**
  312. * Set the quota for a directory.
  313. *
  314. * Quota parameters may have three types of values:
  315. *
  316. * (1) 0 or more: Quota will be set to that value.
  317. * (2) QUOTA_DONT_SET: Quota will not be changed,
  318. * (3) QUOTA_RESET: Quota will be reset.
  319. *
  320. * Any other value is a runtime error.
  321. */
  322. void setQuota(10: common.RequestContext ctx,
  323. /** Path of the directory. */
  324. 1: string path,
  325. /** Limit on the number of names in the directory. */
  326. 2: i64 namespaceQuota,
  327. /**
  328. * Limit on disk space occupied by all the files in the
  329. * directory.
  330. */
  331. 3: i64 diskspaceQuota) throws (1: common.IOException err),
  332. /**
  333. * Set replication factor for an existing file.
  334. *
  335. * This call just updates the value of the replication factor. The actual
  336. * block replication is not expected to be performed during this method call.
  337. * The blocks will be populated or removed in the background as the result of
  338. * the routine block maintenance procedures.
  339. *
  340. * Returns true if successful, false if file does not exist or is a
  341. * directory.
  342. */
  343. bool setReplication(10: common.RequestContext ctx,
  344. /** Path of the file. */
  345. 1: string path,
  346. /** New replication factor. */
  347. 2: i16 replication) throws (1: common.IOException err),
  348. /**
  349. * Delete a file or directory from the file system.
  350. *
  351. * Any blocks belonging to the deleted files will be garbage-collected.
  352. */
  353. bool unlink(10: common.RequestContext ctx,
  354. /** Path of the file or directory. */
  355. 1: string path,
  356. /** Delete a non-empty directory recursively. */
  357. 2: bool recursive) throws (1: common.IOException err),
  358. /**
  359. * Sets the modification and access time of a file or directory.
  360. *
  361. * Setting *one single time paramater* to -1 means that time parameter
  362. * must not be set by this call.
  363. *
  364. * Setting *both time parameters* to -1 means both of them must be set to
  365. * the current time.
  366. */
  367. void utime(10: common.RequestContext ctx,
  368. /** Path of the file or directory. */
  369. 1: string path,
  370. /** Access time in milliseconds since 1970-01-01 00:00 UTC */
  371. 2: i64 atime,
  372. /** Modification time in milliseconds since 1970-01-01 00:00 UTC */
  373. 3: i64 mtime) throws (1: common.IOException err),
  374. /*
  375. * The following methods are meant to be called by datanodes to advertise
  376. * themselves to the namenode.
  377. */
  378. /**
  379. * Inform the namenode that a datanode process has started.
  380. */
  381. void datanodeUp(/** <host name>:<port number> of the datanode */
  382. 1: string name,
  383. /** the storage id of the datanode */
  384. 2: string storage,
  385. /** Thrift port of the datanode */
  386. 3: i32 thriftPort),
  387. /**
  388. * Inform the namenode that a datanode process has stopped.
  389. */
  390. void datanodeDown(/** <host name>:<port number> of the datanode */
  391. 1: string name,
  392. /** the storage id of the datanode */
  393. 2: string storage,
  394. /** Thrift port of the datanode */
  395. 3: i32 thriftPort),
  396. }
  397. /** Encapsulates a block data transfer with its CRC */
  398. struct BlockData {
  399. /** CRC32 of the data being transfered */
  400. 1: i32 crc,
  401. /** Length of the data being transfered */
  402. 2: i32 length,
  403. /** The data itsef */
  404. 3: binary data
  405. }
  406. /**
  407. * Provides an interface to data nodes, so that clients may read and write
  408. * data blocks.
  409. */
  410. service Datanode {
  411. /**
  412. * Read bytes from a block.
  413. *
  414. * Only 2^31 - 1 bytes may be read on a single call to this method.
  415. */
  416. BlockData readBlock(10: common.RequestContext ctx,
  417. /** Block to be read from. */
  418. 1: Block block,
  419. /** Offset within the block where read must start from. */
  420. 2: i64 offset,
  421. /** Number of bytes to read. */
  422. 3: i32 length) throws (1:common.IOException err)
  423. }