|
|
@@ -1,725 +0,0 @@
|
|
|
-#
|
|
|
-# Autogenerated by Thrift
|
|
|
-#
|
|
|
-# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
|
|
|
-#
|
|
|
-
|
|
|
-from thrift.Thrift import *
|
|
|
-
|
|
|
-from thrift.transport import TTransport
|
|
|
-from thrift.protocol import TBinaryProtocol, TProtocol
|
|
|
-try:
|
|
|
- from thrift.protocol import fastbinary
|
|
|
-except:
|
|
|
- fastbinary = None
|
|
|
-
|
|
|
-
|
|
|
-class State(object):
|
|
|
- SUBMITTED = 1
|
|
|
- RUNNING = 2
|
|
|
- SUCCESS = 3
|
|
|
- ERROR = 4
|
|
|
- FAILURE = 5
|
|
|
-
|
|
|
- _VALUES_TO_NAMES = {
|
|
|
- 1: "SUBMITTED",
|
|
|
- 2: "RUNNING",
|
|
|
- 3: "SUCCESS",
|
|
|
- 4: "ERROR",
|
|
|
- 5: "FAILURE",
|
|
|
- }
|
|
|
-
|
|
|
- _NAMES_TO_VALUES = {
|
|
|
- "SUBMITTED": 1,
|
|
|
- "RUNNING": 2,
|
|
|
- "SUCCESS": 3,
|
|
|
- "ERROR": 4,
|
|
|
- "FAILURE": 5,
|
|
|
- }
|
|
|
-
|
|
|
-class PreFabLocalizedFiles(object):
|
|
|
- """
|
|
|
- jar files that are included in the Hadoop distribution,
|
|
|
- and therefore need not to be copied.
|
|
|
- """
|
|
|
- STREAMING = 1
|
|
|
-
|
|
|
- _VALUES_TO_NAMES = {
|
|
|
- 1: "STREAMING",
|
|
|
- }
|
|
|
-
|
|
|
- _NAMES_TO_VALUES = {
|
|
|
- "STREAMING": 1,
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-class SubmissionHandle(object):
|
|
|
- """
|
|
|
- Attributes:
|
|
|
- - id: Unique id for this instance of job submission.
|
|
|
- In practice this is a primary key in the
|
|
|
- database. This struct exists so that the
|
|
|
- handle can evolve (for example, to multiple
|
|
|
- submission servers).
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.I64, 'id', None, None, ), # 1
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, id=None,):
|
|
|
- self.id = id
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.I64:
|
|
|
- self.id = iprot.readI64();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('SubmissionHandle')
|
|
|
- if self.id != None:
|
|
|
- oprot.writeFieldBegin('id', TType.I64, 1)
|
|
|
- oprot.writeI64(self.id)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
- def validate(self):
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class JobData(object):
|
|
|
- """
|
|
|
- Attributes:
|
|
|
- - hadoop_job_ids: Job ID that the JobTracker has for this job
|
|
|
- - stdout_tail: These are "tails" of the stdout/stderr from the Java process doing the
|
|
|
- job submission.
|
|
|
- - stderr_tail
|
|
|
- - state
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.LIST, 'hadoop_job_ids', (TType.STRING,None), None, ), # 1
|
|
|
- (2, TType.STRING, 'stdout_tail', None, None, ), # 2
|
|
|
- (3, TType.STRING, 'stderr_tail', None, None, ), # 3
|
|
|
- (4, TType.I32, 'state', None, None, ), # 4
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, hadoop_job_ids=None, stdout_tail=None, stderr_tail=None, state=None,):
|
|
|
- self.hadoop_job_ids = hadoop_job_ids
|
|
|
- self.stdout_tail = stdout_tail
|
|
|
- self.stderr_tail = stderr_tail
|
|
|
- self.state = state
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.hadoop_job_ids = []
|
|
|
- (_etype3, _size0) = iprot.readListBegin()
|
|
|
- for _i4 in xrange(_size0):
|
|
|
- _elem5 = iprot.readString();
|
|
|
- self.hadoop_job_ids.append(_elem5)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.stdout_tail = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 3:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.stderr_tail = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 4:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.state = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('JobData')
|
|
|
- if self.hadoop_job_ids != None:
|
|
|
- oprot.writeFieldBegin('hadoop_job_ids', TType.LIST, 1)
|
|
|
- oprot.writeListBegin(TType.STRING, len(self.hadoop_job_ids))
|
|
|
- for iter6 in self.hadoop_job_ids:
|
|
|
- oprot.writeString(iter6)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.stdout_tail != None:
|
|
|
- oprot.writeFieldBegin('stdout_tail', TType.STRING, 2)
|
|
|
- oprot.writeString(self.stdout_tail)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.stderr_tail != None:
|
|
|
- oprot.writeFieldBegin('stderr_tail', TType.STRING, 3)
|
|
|
- oprot.writeString(self.stderr_tail)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.state != None:
|
|
|
- oprot.writeFieldBegin('state', TType.I32, 4)
|
|
|
- oprot.writeI32(self.state)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
- def validate(self):
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class SubmissionError(Exception):
|
|
|
- """
|
|
|
- Attributes:
|
|
|
- - message
|
|
|
- - detail
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.STRING, 'message', None, None, ), # 1
|
|
|
- (2, TType.STRING, 'detail', None, None, ), # 2
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, message=None, detail=None,):
|
|
|
- self.message = message
|
|
|
- self.detail = detail
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.message = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.detail = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('SubmissionError')
|
|
|
- if self.message != None:
|
|
|
- oprot.writeFieldBegin('message', TType.STRING, 1)
|
|
|
- oprot.writeString(self.message)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.detail != None:
|
|
|
- oprot.writeFieldBegin('detail', TType.STRING, 2)
|
|
|
- oprot.writeString(self.detail)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
- def validate(self):
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __str__(self):
|
|
|
- return repr(self)
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class LocalizedFile(object):
|
|
|
- """
|
|
|
- File to be copied in a LocalizeFilesStep
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - target_name: Name of file in current directory.
|
|
|
- Framework has the right to make this a symlink.
|
|
|
- - path_on_hdfs: Exactly one of the following may be set
|
|
|
- - pre_fab_localized_file
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.STRING, 'target_name', None, None, ), # 1
|
|
|
- (2, TType.STRING, 'path_on_hdfs', None, None, ), # 2
|
|
|
- (3, TType.I32, 'pre_fab_localized_file', None, None, ), # 3
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, target_name=None, path_on_hdfs=None, pre_fab_localized_file=None,):
|
|
|
- self.target_name = target_name
|
|
|
- self.path_on_hdfs = path_on_hdfs
|
|
|
- self.pre_fab_localized_file = pre_fab_localized_file
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.target_name = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.path_on_hdfs = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 3:
|
|
|
- if ftype == TType.I32:
|
|
|
- self.pre_fab_localized_file = iprot.readI32();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('LocalizedFile')
|
|
|
- if self.target_name != None:
|
|
|
- oprot.writeFieldBegin('target_name', TType.STRING, 1)
|
|
|
- oprot.writeString(self.target_name)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.path_on_hdfs != None:
|
|
|
- oprot.writeFieldBegin('path_on_hdfs', TType.STRING, 2)
|
|
|
- oprot.writeString(self.path_on_hdfs)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.pre_fab_localized_file != None:
|
|
|
- oprot.writeFieldBegin('pre_fab_localized_file', TType.I32, 3)
|
|
|
- oprot.writeI32(self.pre_fab_localized_file)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
- def validate(self):
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class LocalizeFilesStep(object):
|
|
|
- """
|
|
|
- Places files into the working directory of "jobsub plan execution".
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - localize_files
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.LIST, 'localize_files', (TType.STRUCT,(LocalizedFile, LocalizedFile.thrift_spec)), None, ), # 1
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, localize_files=None,):
|
|
|
- self.localize_files = localize_files
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.localize_files = []
|
|
|
- (_etype10, _size7) = iprot.readListBegin()
|
|
|
- for _i11 in xrange(_size7):
|
|
|
- _elem12 = LocalizedFile()
|
|
|
- _elem12.read(iprot)
|
|
|
- self.localize_files.append(_elem12)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('LocalizeFilesStep')
|
|
|
- if self.localize_files != None:
|
|
|
- oprot.writeFieldBegin('localize_files', TType.LIST, 1)
|
|
|
- oprot.writeListBegin(TType.STRUCT, len(self.localize_files))
|
|
|
- for iter13 in self.localize_files:
|
|
|
- iter13.write(oprot)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
- def validate(self):
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class BinHadoopStep(object):
|
|
|
- """
|
|
|
- Runs bin/hadoop, with enough environment to point at the
|
|
|
- configured cluster, and to assume the correct user.
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - arguments
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.LIST, 'arguments', (TType.STRING,None), None, ), # 1
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, arguments=None,):
|
|
|
- self.arguments = arguments
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.arguments = []
|
|
|
- (_etype17, _size14) = iprot.readListBegin()
|
|
|
- for _i18 in xrange(_size14):
|
|
|
- _elem19 = iprot.readString();
|
|
|
- self.arguments.append(_elem19)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('BinHadoopStep')
|
|
|
- if self.arguments != None:
|
|
|
- oprot.writeFieldBegin('arguments', TType.LIST, 1)
|
|
|
- oprot.writeListBegin(TType.STRING, len(self.arguments))
|
|
|
- for iter20 in self.arguments:
|
|
|
- oprot.writeString(iter20)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
- def validate(self):
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class SubmissionPlanStep(object):
|
|
|
- """
|
|
|
- A union of all possible steps.
|
|
|
-
|
|
|
- Note: THRIFT-409 (committed Sep 1 09) added support for unions
|
|
|
- to Thrift. After a thrift release, we could switch to that syntax.
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - localize_files_step: Only one of these fields may be specified!
|
|
|
- - bin_hadoop_step
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.STRUCT, 'localize_files_step', (LocalizeFilesStep, LocalizeFilesStep.thrift_spec), None, ), # 1
|
|
|
- (2, TType.STRUCT, 'bin_hadoop_step', (BinHadoopStep, BinHadoopStep.thrift_spec), None, ), # 2
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, localize_files_step=None, bin_hadoop_step=None,):
|
|
|
- self.localize_files_step = localize_files_step
|
|
|
- self.bin_hadoop_step = bin_hadoop_step
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.STRUCT:
|
|
|
- self.localize_files_step = LocalizeFilesStep()
|
|
|
- self.localize_files_step.read(iprot)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.STRUCT:
|
|
|
- self.bin_hadoop_step = BinHadoopStep()
|
|
|
- self.bin_hadoop_step.read(iprot)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('SubmissionPlanStep')
|
|
|
- if self.localize_files_step != None:
|
|
|
- oprot.writeFieldBegin('localize_files_step', TType.STRUCT, 1)
|
|
|
- self.localize_files_step.write(oprot)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.bin_hadoop_step != None:
|
|
|
- oprot.writeFieldBegin('bin_hadoop_step', TType.STRUCT, 2)
|
|
|
- self.bin_hadoop_step.write(oprot)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
- def validate(self):
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|
|
|
-
|
|
|
-class SubmissionPlan(object):
|
|
|
- """
|
|
|
- Plan to be executed by jobsub.
|
|
|
-
|
|
|
- Attributes:
|
|
|
- - name
|
|
|
- - user
|
|
|
- - groups
|
|
|
- - steps: Steps to execute, in order.
|
|
|
- - save_output: Directory name in HDFS where stdout and stderr will be put
|
|
|
- """
|
|
|
-
|
|
|
- thrift_spec = (
|
|
|
- None, # 0
|
|
|
- (1, TType.STRING, 'name', None, None, ), # 1
|
|
|
- (2, TType.STRING, 'user', None, None, ), # 2
|
|
|
- (3, TType.LIST, 'groups', (TType.STRING,None), None, ), # 3
|
|
|
- (4, TType.LIST, 'steps', (TType.STRUCT,(SubmissionPlanStep, SubmissionPlanStep.thrift_spec)), None, ), # 4
|
|
|
- (5, TType.STRING, 'save_output', None, None, ), # 5
|
|
|
- )
|
|
|
-
|
|
|
- def __init__(self, name=None, user=None, groups=None, steps=None, save_output=None,):
|
|
|
- self.name = name
|
|
|
- self.user = user
|
|
|
- self.groups = groups
|
|
|
- self.steps = steps
|
|
|
- self.save_output = save_output
|
|
|
-
|
|
|
- def read(self, iprot):
|
|
|
- if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
|
|
|
- return
|
|
|
- iprot.readStructBegin()
|
|
|
- while True:
|
|
|
- (fname, ftype, fid) = iprot.readFieldBegin()
|
|
|
- if ftype == TType.STOP:
|
|
|
- break
|
|
|
- if fid == 1:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.name = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 2:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.user = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 3:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.groups = []
|
|
|
- (_etype24, _size21) = iprot.readListBegin()
|
|
|
- for _i25 in xrange(_size21):
|
|
|
- _elem26 = iprot.readString();
|
|
|
- self.groups.append(_elem26)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 4:
|
|
|
- if ftype == TType.LIST:
|
|
|
- self.steps = []
|
|
|
- (_etype30, _size27) = iprot.readListBegin()
|
|
|
- for _i31 in xrange(_size27):
|
|
|
- _elem32 = SubmissionPlanStep()
|
|
|
- _elem32.read(iprot)
|
|
|
- self.steps.append(_elem32)
|
|
|
- iprot.readListEnd()
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- elif fid == 5:
|
|
|
- if ftype == TType.STRING:
|
|
|
- self.save_output = iprot.readString();
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- else:
|
|
|
- iprot.skip(ftype)
|
|
|
- iprot.readFieldEnd()
|
|
|
- iprot.readStructEnd()
|
|
|
-
|
|
|
- def write(self, oprot):
|
|
|
- if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
|
|
|
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
|
|
|
- return
|
|
|
- oprot.writeStructBegin('SubmissionPlan')
|
|
|
- if self.name != None:
|
|
|
- oprot.writeFieldBegin('name', TType.STRING, 1)
|
|
|
- oprot.writeString(self.name)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.user != None:
|
|
|
- oprot.writeFieldBegin('user', TType.STRING, 2)
|
|
|
- oprot.writeString(self.user)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.groups != None:
|
|
|
- oprot.writeFieldBegin('groups', TType.LIST, 3)
|
|
|
- oprot.writeListBegin(TType.STRING, len(self.groups))
|
|
|
- for iter33 in self.groups:
|
|
|
- oprot.writeString(iter33)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.steps != None:
|
|
|
- oprot.writeFieldBegin('steps', TType.LIST, 4)
|
|
|
- oprot.writeListBegin(TType.STRUCT, len(self.steps))
|
|
|
- for iter34 in self.steps:
|
|
|
- iter34.write(oprot)
|
|
|
- oprot.writeListEnd()
|
|
|
- oprot.writeFieldEnd()
|
|
|
- if self.save_output != None:
|
|
|
- oprot.writeFieldBegin('save_output', TType.STRING, 5)
|
|
|
- oprot.writeString(self.save_output)
|
|
|
- oprot.writeFieldEnd()
|
|
|
- oprot.writeFieldStop()
|
|
|
- oprot.writeStructEnd()
|
|
|
- def validate(self):
|
|
|
- return
|
|
|
-
|
|
|
-
|
|
|
- def __repr__(self):
|
|
|
- L = ['%s=%r' % (key, value)
|
|
|
- for key, value in self.__dict__.iteritems()]
|
|
|
- return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
|
-
|
|
|
- def __eq__(self, other):
|
|
|
- return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
|
-
|
|
|
- def __ne__(self, other):
|
|
|
- return not (self == other)
|