jobtracker.thrift 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484
  1. /*
  2. * Licensed to Cloudera, Inc. under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. Cloudera, Inc. licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. /*
  19. * Thrift interface for Hadoop JobTracker.
  20. */
  21. /* Common types and interfaces */
  22. include 'common.thrift'
  23. /*
  24. * Namespaces for generated code. The idea is to keep code generated by
  25. * Thrift under a 'hadoop.api' namespace, so that a higher-level set of
  26. * functions and classes may be defined under 'hadoop'.
  27. */
  28. namespace cpp hadoop.api.jobtracker
  29. namespace csharp Hadoop.API.JobTracker
  30. namespace java org.apache.hadoop.thriftfs.jobtracker.api
  31. namespace perl Hadoop.API.jobtracker
  32. namespace php hadoop_api_jobtracker
  33. namespace py hadoop.api.jobtracker
  34. namespace rb Hadoop.API.jobtracker
  35. /*
  36. * All type names are prefixed with 'Thrift' to avoid confusion when dealing
  37. * with both thrift and hadoop-land objects with the same name in Java.
  38. */
  39. enum ThriftTaskType {
  40. MAP,
  41. REDUCE,
  42. JOB_SETUP,
  43. JOB_CLEANUP,
  44. TASK_CLEANUP // What is this?
  45. }
  46. enum ThriftTaskState {
  47. RUNNING, SUCCEEDED, FAILED, UNASSIGNED, KILLED,
  48. COMMIT_PENDING, FAILED_UNCLEAN, KILLED_UNCLEAN
  49. }
  50. enum ThriftTaskPhase {
  51. STARTING, MAP, SHUFFLE, SORT, REDUCE, CLEANUP
  52. }
  53. /**
  54. * It corresponds to the (inferred) internal state of a TaskInProgress,
  55. * and not that of a TaskStatus.
  56. */
  57. enum ThriftTaskQueryState {
  58. SUCCEEDED,
  59. FAILED,
  60. RUNNING, /* Inferred - Only if startTime is set */
  61. PENDING, /* Inferred - Only if startTime is not set */
  62. KILLED
  63. }
  64. /** Possible job priorities (see ThriftJobStatus) */
  65. enum ThriftJobPriority {
  66. VERY_HIGH,
  67. HIGH,
  68. NORMAL,
  69. LOW,
  70. VERY_LOW
  71. }
  72. /** Unique identifier for each job */
  73. struct ThriftJobID {
  74. /** Unique id of jobtracker */
  75. 1: string jobTrackerID
  76. /** Unique (to JT) job id */
  77. 2: i32 jobID
  78. /** Flattened as a string */
  79. 3: string asString
  80. }
  81. /** Description of a job queue */
  82. struct ThriftJobQueueInfo {
  83. 1: string queueName
  84. 2: string schedulingInfo
  85. }
  86. struct ThriftJobQueueList {
  87. 1: list<ThriftJobQueueInfo> queues
  88. }
  89. /** Counter which represents some custom job metric */
  90. struct ThriftCounter {
  91. 1: string name
  92. 2: string displayName
  93. 3: i64 value
  94. }
  95. /** Counters are organized by group */
  96. struct ThriftCounterGroup {
  97. 1: string name
  98. 2: string displayName
  99. 3: map<string, ThriftCounter> counters
  100. }
  101. /** Container structure for counter groups */
  102. struct ThriftGroupList {
  103. 1: list<ThriftCounterGroup> groups
  104. }
  105. /** Counters for map tasks only, reduce tasks only, and job-scoped counters */
  106. struct ThriftJobCounterRollups {
  107. 1: ThriftGroupList mapCounters; // eg map input bytes
  108. 2: ThriftGroupList reduceCounters; // eg reduce input bytes
  109. 3: ThriftGroupList jobCounters; // eg task counts, etc
  110. }
  111. /** Unique task id */
  112. struct ThriftTaskID {
  113. /** ID of the job to which the task belongs */
  114. 1: ThriftJobID jobID
  115. /** What kind of task is this? */
  116. 2: ThriftTaskType taskType
  117. /** Unique (to job) task id */
  118. 3: i32 taskID
  119. /** Flattened to a unique string */
  120. 4: string asString
  121. }
  122. /** Unique task attempt id */
  123. struct ThriftTaskAttemptID {
  124. 1: ThriftTaskID taskID
  125. 2: i32 attemptID
  126. 3: string asString
  127. }
  128. /** Describes the current state of a single attempt */
  129. struct ThriftTaskStatus {
  130. 1: ThriftTaskAttemptID taskID
  131. 2: double progress
  132. 3: ThriftTaskState state
  133. 4: string diagnosticInfo
  134. 5: string stateString
  135. 6: string taskTracker
  136. 7: i64 startTime
  137. 8: i64 finishTime
  138. 9: i64 outputSize
  139. 10: ThriftTaskPhase phase
  140. 11: ThriftGroupList counters
  141. 12: i64 shuffleFinishTime,
  142. 13: i64 sortFinishTime,
  143. 14: i64 mapFinishTime,
  144. }
  145. /**
  146. * A ThriftTaskInProgress contains a list of
  147. * task attempts (speculatively executed instances of the same task).
  148. * These are indexed by TaskAttemptID.
  149. * For simplicity, we convert maps keyed on TaskAttemptIDs to maps keyed
  150. * on their string representation.
  151. *
  152. * Assumption: there won't be so many task attempts that retrieving a single task
  153. * will be too expensive.
  154. */
  155. struct ThriftTaskInProgress {
  156. 2: i64 execStartTime
  157. 3: i64 execFinishTime
  158. 4: double progress
  159. 5: i64 startTime
  160. 6: bool failed
  161. 7: bool complete
  162. 8: ThriftTaskID taskID
  163. 9: list<ThriftTaskAttemptID> tasks
  164. /** TaskAttemptID (string) to ThriftTaskStatus map */
  165. 10: map<string,ThriftTaskStatus> taskStatuses
  166. 11: map<string,list<string>> taskDiagnosticData
  167. 12: ThriftGroupList counters
  168. /* The last state reported (from TaskReport) */
  169. 13: string mostRecentState
  170. /* The set of attempts that are currently running - could be empty. */
  171. 14: list<string> runningAttempts
  172. /* The id of the successful attempt. If not complete, this field is meaningless */
  173. 15: string successfulAttempt
  174. }
  175. /** TaskTracker status; contains details of individual tasks */
  176. struct ThriftTaskTrackerStatus {
  177. 1: string trackerName
  178. 2: string host
  179. 3: i32 httpPort
  180. 4: i32 failureCount
  181. /** List of the state of all tasks on this tracker */
  182. 5: list<ThriftTaskStatus> taskReports
  183. /** When did the JobTracker last hear from this TaskTracker? */
  184. 6: i64 lastSeen
  185. /** Maximum possible number of both task types */
  186. 7: i32 maxMapTasks
  187. 8: i32 maxReduceTasks
  188. /** Main memory metrics, all in bytes */
  189. 9: i64 totalVirtualMemory
  190. 11: i64 totalPhysicalMemory
  191. 13: i64 availableSpace
  192. /** Currently running and unassigned map and reduce tasks */
  193. 14: i32 mapCount
  194. 15: i32 reduceCount
  195. }
  196. /** Container structure for TaskTrackerStatus objects */
  197. struct ThriftTaskTrackerStatusList {
  198. 1: list<ThriftTaskTrackerStatus> trackers
  199. }
  200. /** States that the jobtracker may be in */
  201. enum JobTrackerState {
  202. INITIALIZING,
  203. RUNNING
  204. }
  205. /** Enum version of the ints in JobStatus */
  206. enum ThriftJobState {
  207. RUNNING = 1,
  208. SUCCEEDED = 2,
  209. FAILED = 3,
  210. PREP = 4,
  211. KILLED = 5
  212. }
  213. /** Status of a job */
  214. struct ThriftJobStatus {
  215. 1: ThriftJobID jobID
  216. 2: double mapProgress
  217. 3: double reduceProgress
  218. 4: double cleanupProgress
  219. 5: double setupProgress
  220. 6: ThriftJobState runState
  221. 7: i64 startTime
  222. 8: string user
  223. 9: ThriftJobPriority priority
  224. 10: string schedulingInfo
  225. }
  226. /** Job metadata */
  227. struct ThriftJobProfile {
  228. 1: string user
  229. 2: ThriftJobID jobID
  230. 3: string jobFile
  231. 4: string name
  232. 5: string queueName
  233. }
  234. /**
  235. * Container structure of a list of tasks. This list may have been put together
  236. * according to some selection criteria. That is, it may not correspond to the
  237. * mapTasks, or reduceTasks, etc. It may even contain tasks of different types.
  238. */
  239. struct ThriftTaskInProgressList {
  240. /** A (possibly incomplete) list of tasks */
  241. 1: list<ThriftTaskInProgress> tasks
  242. /** The total number of tasks in this full list. */
  243. 2: i32 numTotalTasks
  244. }
  245. /** Status of *all* jobs, not just currently running ones */
  246. struct ThriftJobInProgress {
  247. 1: ThriftJobProfile profile
  248. 2: ThriftJobStatus status
  249. 3: ThriftJobID jobID
  250. 4: i32 desiredMaps /* N/A for a Retired job */
  251. 5: i32 desiredReduces /* N/A for a Retired job */
  252. 6: i32 finishedMaps /* N/A for a Retired job */
  253. 7: i32 finishedReduces /* N/A for a Retired job */
  254. 8: ThriftJobPriority priority /* N/A for a Retired job, but present in status field */
  255. 11: i64 startTime
  256. 12: i64 finishTime /* N/A for a Retired job */
  257. 13: i64 launchTime /* N/A for a Retired job */
  258. 23: ThriftTaskInProgressList tasks /* N/A for a Retired job */
  259. }
  260. /** Container structure of a list of jobs, in case we ever want to add metadata */
  261. struct ThriftJobList {
  262. 1: list<ThriftJobInProgress> jobs
  263. }
  264. /** Container structure for job counts for a given user */
  265. struct ThriftUserJobCounts {
  266. 1: i32 nPrep,
  267. 2: i32 nRunning,
  268. 3: i32 nSucceeded,
  269. 4: i32 nFailed,
  270. 5: i32 nKilled
  271. }
  272. /** Status of the cluster as viewed by the jobtracker */
  273. struct ThriftClusterStatus {
  274. 1: i32 numActiveTrackers
  275. 2: list<string> activeTrackerNames
  276. 3: list<string> blacklistedTrackerNames
  277. 4: i32 numBlacklistedTrackers
  278. 5: i32 numExcludedNodes
  279. /* How often does the JobTracker check for expired tasks with the taskTracker */
  280. 6: i64 taskTrackerExpiryInterval
  281. 7: i32 mapTasks;
  282. 8: i32 reduceTasks
  283. 9: i32 maxMapTasks
  284. 10: i32 maxReduceTasks
  285. 11: JobTrackerState state
  286. /** Used and max memory for the cluster, in bytes */
  287. 12: i64 usedMemory
  288. 13: i64 maxMemory
  289. 14: i32 totalSubmissions
  290. /* True if the JobTracker has restarted */
  291. 15: bool hasRestarted
  292. /* True if the JobTracker has finished recovering after a restart */
  293. 16: bool hasRecovered
  294. 17: i64 startTime
  295. 18: string hostname
  296. 19: string identifier
  297. 20: i32 httpPort
  298. }
  299. /** Merely an indicator that job wasn't found. */
  300. exception JobNotFoundException {
  301. }
  302. /** Merely an indicator that task wasn't found. */
  303. exception TaskNotFoundException {
  304. }
  305. /** Indicates that a task attempt wasn't found */
  306. exception TaskAttemptNotFoundException {
  307. }
  308. /** Indicates that a tasktracker wasn't found */
  309. exception TaskTrackerNotFoundException {
  310. }
  311. /** A proxy service onto a Jobtracker, exposing read-only methods for cluster monitoring */
  312. service Jobtracker extends common.HadoopServiceBase {
  313. /** Get the name of the tracker exporting this service */
  314. string getJobTrackerName(10: common.RequestContext ctx),
  315. /** Get the current cluster status */
  316. ThriftClusterStatus getClusterStatus(10: common.RequestContext ctx),
  317. /** Get a list of job queues managed by this tracker */
  318. ThriftJobQueueList getQueues(10: common.RequestContext ctx)
  319. throws(1: common.IOException err),
  320. /** Get a job by ID */
  321. ThriftJobInProgress getJob(10: common.RequestContext ctx, 1: ThriftJobID jobID)
  322. throws(1: JobNotFoundException err),
  323. /** Get a list of currently running jobs */
  324. ThriftJobList getRunningJobs(10: common.RequestContext ctx),
  325. /** Get a list of completed jobs */
  326. ThriftJobList getCompletedJobs(10: common.RequestContext ctx),
  327. /** Get a retired job */
  328. ThriftJobInProgress getRetiredJob(10: common.RequestContext ctx, 1: ThriftJobID jobID)
  329. throws(1: JobNotFoundException err),
  330. /** Get a list of retired jobs */
  331. ThriftJobList getRetiredJobs(10: common.RequestContext ctx, 1: ThriftJobState state),
  332. /** Get a list of failed (due to error, not killed) jobs */
  333. ThriftJobList getFailedJobs(10: common.RequestContext ctx),
  334. /** Get a list of killed jobs */
  335. ThriftJobList getKilledJobs(10: common.RequestContext ctx),
  336. /** Get a list of all failed, completed and running jobs (could be expensive!) */
  337. ThriftJobList getAllJobs(10: common.RequestContext ctx),
  338. /** Get the count of jobs by status for a given user */
  339. ThriftUserJobCounts getUserJobCounts(1: common.RequestContext ctx, 2: string user),
  340. /** Get a (possibly incomplete) list of tasks */
  341. ThriftTaskInProgressList getTaskList(
  342. 1: common.RequestContext ctx,
  343. 2: ThriftJobID jobID,
  344. 3: set<ThriftTaskType> types,
  345. 4: set<ThriftTaskQueryState> states,
  346. 5: string text,
  347. 6: i32 count,
  348. 7: i32 offset) throws(1: JobNotFoundException err),
  349. /** Get details of a task */
  350. ThriftTaskInProgress getTask(1: common.RequestContext ctx,
  351. 2: ThriftTaskID taskID)
  352. throws(1: JobNotFoundException jnf, 2: TaskNotFoundException tnf),
  353. /**
  354. * Get a list of groups of counters attached to the job with provided id.
  355. * This returns the total counters
  356. **/
  357. ThriftGroupList getJobCounters(10: common.RequestContext ctx,
  358. 1: ThriftJobID jobID)
  359. throws(1: JobNotFoundException err),
  360. /** Return job counters rolled up by map, reduce, and total */
  361. ThriftJobCounterRollups getJobCounterRollups(10: common.RequestContext ctx,
  362. 1: ThriftJobID jobID)
  363. throws(1: JobNotFoundException err),
  364. /** Get all active trackers */
  365. ThriftTaskTrackerStatusList getActiveTrackers(10: common.RequestContext ctx),
  366. /** Get all blacklisted trackers */
  367. ThriftTaskTrackerStatusList getBlacklistedTrackers(10: common.RequestContext ctx),
  368. /** Get all trackers */
  369. ThriftTaskTrackerStatusList getAllTrackers(10: common.RequestContext ctx),
  370. /** Get a single task tracker by name */
  371. ThriftTaskTrackerStatus getTracker(10: common.RequestContext ctx, 1: string name)
  372. throws(1: TaskTrackerNotFoundException tne),
  373. /** Get the current time in ms according to the JT */
  374. i64 getCurrentTime(10: common.RequestContext ctx),
  375. /** Get the xml for a job's configuration, serialised from the local filesystem on the JT */
  376. string getJobConfXML(10: common.RequestContext ctx, 1: ThriftJobID jobID)
  377. throws(1: common.IOException err),
  378. /** Kill a job */
  379. void killJob(10: common.RequestContext ctx, 1: ThriftJobID jobID)
  380. throws(1: common.IOException err, 2: JobNotFoundException jne),
  381. /** Kill a task attempt */
  382. void killTaskAttempt(10: common.RequestContext ctx, 1: ThriftTaskAttemptID attemptID)
  383. throws(1: common.IOException err,
  384. 2: TaskAttemptNotFoundException tne,
  385. 3: JobNotFoundException jne),
  386. /** Set a job's priority */
  387. void setJobPriority(10: common.RequestContext ctx,
  388. 1: ThriftJobID jobID,
  389. 2: ThriftJobPriority priority)
  390. throws(1: common.IOException err, 2: JobNotFoundException jne),
  391. /** Get an MR delegation token. */
  392. common.ThriftDelegationToken getDelegationToken(10:common.RequestContext ctx, 1:string renewer)
  393. throws(1: common.IOException err)
  394. }