framework.py 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197
  1. # Licensed to Cloudera, Inc. under one
  2. # or more contributor license agreements. See the NOTICE file
  3. # distributed with this work for additional information
  4. # regarding copyright ownership. Cloudera, Inc. licenses this file
  5. # to you under the Apache License, Version 2.0 (the
  6. # "License"); you may not use this file except in compliance
  7. # with the License. You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import logging
  17. from form import Form
  18. class Framework(object):
  19. """
  20. Sqoop framework object.
  21. Example of sqoop framework dictionary received by server: {
  22. "id": 1,
  23. "resources": {
  24. "output.label": "Output configuration",
  25. "security.maxConnections.help": "Maximal number of connections that this connection object can use at one point in time",
  26. "output.storageType.label": "Storage type",
  27. "output.ignored.help": "This value is ignored",
  28. "input.label": "Input configuration",
  29. "security.help": "You must supply the information requested in order to create a job object.",
  30. "output.storageType.help": "Target on Hadoop ecosystem where to store data",
  31. "input.inputDirectory.help": "Directory that should be exported",
  32. "output.outputFormat.label": "Output format",
  33. "output.ignored.label": "Ignored",
  34. "output.outputFormat.help": "Format in which data should be serialized",
  35. "output.help": "You must supply the information requested in order to get information where you want to store your data.",
  36. "throttling.help": "Set throttling boundaries to not overload your systems",
  37. "input.inputDirectory.label": "Input directory",
  38. "throttling.loaders.label": "Loaders",
  39. "input.help": "Specifies information required to get data from Hadoop ecosystem",
  40. "throttling.extractors.label": "Extractors",
  41. "throttling.extractors.help": "Number of extractors that Sqoop will use",
  42. "security.label": "Security related configuration options",
  43. "throttling.label": "Throttling resources",
  44. "throttling.loaders.help": "Number of loaders that Sqoop will use",
  45. "output.outputDirectory.help": "Output directory for final data",
  46. "security.maxConnections.label": "Max connections",
  47. "output.outputDirectory.label": "Output directory"
  48. },
  49. "job-forms": {
  50. "IMPORT": [
  51. {
  52. "id": 7,
  53. "inputs": [
  54. {
  55. "id": 20,
  56. "values": "HDFS",
  57. "name": "output.storageType",
  58. "type": "ENUM",
  59. "sensitive": false
  60. },
  61. {
  62. "id": 21,
  63. "values": "TEXT_FILE,SEQUENCE_FILE",
  64. "name": "output.outputFormat",
  65. "type": "ENUM",
  66. "sensitive": false
  67. },
  68. {
  69. "id": 22,
  70. "name": "output.outputDirectory",
  71. "type": "STRING",
  72. "size": 255,
  73. "sensitive": false
  74. }
  75. ],
  76. "name": "output",
  77. "type": "CONNECTION"
  78. },
  79. {
  80. "id": 8,
  81. "inputs": [
  82. {
  83. "id": 23,
  84. "name": "throttling.extractors",
  85. "type": "INTEGER",
  86. "sensitive": false
  87. },
  88. {
  89. "id": 24,
  90. "name": "throttling.loaders",
  91. "type": "INTEGER",
  92. "sensitive": false
  93. }
  94. ],
  95. "name": "throttling",
  96. "type": "CONNECTION"
  97. }
  98. ],
  99. "EXPORT": [
  100. {
  101. "id": 5,
  102. "inputs": [
  103. {
  104. "id": 17,
  105. "name": "input.inputDirectory",
  106. "type": "STRING",
  107. "size": 255,
  108. "sensitive": false
  109. }
  110. ],
  111. "name": "input",
  112. "type": "CONNECTION"
  113. },
  114. {
  115. "id": 6,
  116. "inputs": [
  117. {
  118. "id": 18,
  119. "name": "throttling.extractors",
  120. "type": "INTEGER",
  121. "sensitive": false
  122. },
  123. {
  124. "id": 19,
  125. "name": "throttling.loaders",
  126. "type": "INTEGER",
  127. "sensitive": false
  128. }
  129. ],
  130. "name": "throttling",
  131. "type": "CONNECTION"
  132. }
  133. ]
  134. },
  135. "con-forms": [
  136. {
  137. "id": 4,
  138. "inputs": [
  139. {
  140. "id": 16,
  141. "name": "security.maxConnections",
  142. "type": "INTEGER",
  143. "sensitive": false
  144. }
  145. ],
  146. "name": "security",
  147. "type": "CONNECTION"
  148. }
  149. ]
  150. }
  151. The ``job-forms`` and ``con-forms`` keys hold forms.
  152. The ``job-forms`` key will hold 2 sets of forms: IMPORT and EXPORT.
  153. ``id`` is for look up in metadata repository.
  154. The framework API will return resource information.
  155. The keys are names associated with inputs in the various forms.
  156. @see sqoop.client.form for more information on unstructured forms in sqoop.
  157. """
  158. def __init__(self, id, job_forms, con_forms, resources, **kwargs):
  159. self.id = id
  160. self.job_forms = job_forms
  161. self.con_forms = con_forms
  162. self.resources = resources
  163. @staticmethod
  164. def from_dict(framework_dict):
  165. framework_dict.setdefault('job-forms', {})
  166. framework_dict['job_forms'] = {}
  167. if 'IMPORT' in framework_dict['job-forms']:
  168. framework_dict['job_forms']['IMPORT'] = [ Form.from_dict(job_form_dict) for job_form_dict in framework_dict['job-forms']['IMPORT'] ]
  169. if 'EXPORT' in framework_dict['job-forms']:
  170. framework_dict['job_forms']['EXPORT'] = [ Form.from_dict(job_form_dict) for job_form_dict in framework_dict['job-forms']['EXPORT'] ]
  171. framework_dict.setdefault('con-forms', [])
  172. framework_dict['con_forms'] = [ Form.from_dict(con_form_dict) for con_form_dict in framework_dict['con-forms'] ]
  173. return Framework(**framework_dict)
  174. def to_dict(self):
  175. d = {
  176. 'id': self.id,
  177. 'con-forms': [ con_form.to_dict() for con_form in self.con_forms ],
  178. 'job-forms': {},
  179. 'resources': self.resources
  180. }
  181. if 'IMPORT' in self.job_forms:
  182. d['job-forms']['IMPORT'] = [ job_form.to_dict() for job_form in self.job_forms['IMPORT'] ]
  183. if 'EXPORT' in self.job_forms:
  184. d['job-forms']['EXPORT'] = [ job_form.to_dict() for job_form in self.job_forms['EXPORT'] ]
  185. return d