generate_requirements.py 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219
  1. """
  2. Dynamic requirements generator for Hue multi-python build system.
  3. Generates architecture- and Python-version-specific requirement files
  4. at build time. This tool replaces static requirements.txt and supports:
  5. - Local editable paths for Hue extensions.
  6. - Binary wheels for specific platforms (e.g., ppc64le).
  7. - Architecture-aware dependency lists.
  8. """
  9. import os
  10. import sys
  11. import shutil
  12. import platform
  13. this_dir = os.path.dirname(os.path.abspath(__file__))
  14. class RequirementsGenerator:
  15. def __init__(self):
  16. self.local_requirements = [
  17. "boto-2.49.0",
  18. "django-axes-5.13.0",
  19. "django-babel",
  20. "pysaml2-7.3.1",
  21. "python-sasl-0.3.1",
  22. ]
  23. self.requirements = [
  24. "setuptools==70.0.0",
  25. "apache-ranger==0.0.3",
  26. "requests-gssapi==1.2.3",
  27. "asn1crypto==0.24.0",
  28. "avro-python3==1.8.2",
  29. "Babel==2.9.1",
  30. "celery[redis]==5.4.0",
  31. "cffi==1.15.0",
  32. "channels==4.0.0",
  33. "channels-redis==4.0.0",
  34. "configobj==5.0.9",
  35. "cx-Oracle==8.3.0",
  36. "django-auth-ldap==4.3.0",
  37. "Django==4.1.13",
  38. "daphne==3.0.2",
  39. "django-redis==5.4.0",
  40. "django-celery-beat==2.6.0",
  41. "django-celery-results==2.5.1",
  42. "django-cors-headers==3.13.0",
  43. "django-crequest==2018.5.11",
  44. "django-debug-panel==0.8.3",
  45. "django-debug-toolbar==3.6.0",
  46. "django-extensions==3.2.1",
  47. "django-ipware==3.0.2",
  48. "django_opentracing==1.1.0",
  49. "django_prometheus==1.0.15",
  50. "django-webpack-loader==1.0.0",
  51. "djangomako==1.3.2",
  52. "djangorestframework-simplejwt==5.2.1",
  53. "djangorestframework==3.14.0",
  54. "djangosaml2==1.9.3",
  55. "future==0.18.3",
  56. "gcs-oauth2-boto-plugin==3.0",
  57. "greenlet==3.1.1",
  58. "gunicorn==23.0.0",
  59. "ipython==8.12.2", # Python >= 3.8
  60. "jaeger-client==4.3.0",
  61. "jdcal==1.0.1",
  62. "kazoo==2.8.0",
  63. "kerberos==1.3.0",
  64. "kubernetes==26.1.0",
  65. "Mako==1.2.3",
  66. "openpyxl==3.0.9",
  67. "phoenixdb==1.2.1",
  68. "prompt-toolkit==3.0.39",
  69. "protobuf==3.20.3",
  70. "pyarrow==17.0.0",
  71. "pyformance==0.3.2",
  72. "python-dateutil==2.8.2",
  73. "python-daemon==2.2.4",
  74. "python-ldap==3.4.3",
  75. "python-oauth2==1.1.0",
  76. "python-pam==2.0.2",
  77. "pytidylib==0.3.2",
  78. "pytz==2021.3",
  79. "PyJWT==2.4.0",
  80. "PyYAML==6.0.1",
  81. "requests==2.32.3",
  82. "requests-kerberos==0.14.0",
  83. "krb5==0.5.1", # pinned for Sles12, dep of requests-kerberos 0.14.0
  84. "rsa==4.7.2",
  85. "ruff==0.4.2",
  86. "slack-sdk==3.31.0",
  87. "SQLAlchemy==1.3.8",
  88. "sqlparse==0.5.0",
  89. "tablib==0.13.0",
  90. "tabulate==0.8.9",
  91. "trino==0.329.0",
  92. "git+https://github.com/gethue/thrift.git",
  93. "thrift-sasl==0.4.3",
  94. "django-utils-six==2.0",
  95. "six==1.16.0",
  96. "psutil==5.8.0",
  97. "drf-spectacular[sidecar]==0.27.2",
  98. ]
  99. self.ppc64le_requirements = {
  100. "default": [],
  101. "3.8": [
  102. "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/cryptography-41.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
  103. "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/numpy-1.23.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
  104. "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/pandas-1.4.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
  105. "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/lxml-4.6.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
  106. "PyYAML==5.4.1",
  107. "Markdown==3.1",
  108. ],
  109. "3.9": [
  110. "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/cryptography-41.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
  111. "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/numpy-1.23.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
  112. "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/pandas-1.4.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
  113. "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/lxml-4.6.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
  114. "PyYAML==6.0.1",
  115. "Markdown==3.8",
  116. ]
  117. }
  118. self.x86_64_requirements = {
  119. "default": [
  120. "cryptography==42.0.8",
  121. "numpy==1.24.4",
  122. "pandas==2.0.3",
  123. "lxml==4.9.1",
  124. "sasl==0.3.1",
  125. "Markdown==3.1",
  126. ],
  127. "3.9": [
  128. "pyopenssl==22.1.0",
  129. "numpy==1.24.4",
  130. "pandas==2.0.3",
  131. "lxml==4.9.1",
  132. "sasl==0.3.1",
  133. "Markdown==3.8",
  134. ],
  135. "3.11": [
  136. "cryptography==42.0.8",
  137. "numpy==1.24.4",
  138. "pandas==2.0.3",
  139. "lxml==4.9.1",
  140. "async-timeout==5.0.1",
  141. "pure-sasl==0.6.2",
  142. "Markdown==3.8",
  143. ],
  144. }
  145. self.aarch64_requirements = {
  146. "default": [
  147. "cryptography==42.0.8",
  148. "numpy==1.24.4",
  149. "pandas==2.0.3",
  150. "lxml==4.9.1",
  151. "Markdown==3.1",
  152. ],
  153. "3.9": [
  154. "pyopenssl==22.1.0",
  155. "numpy==1.24.4",
  156. "pandas==2.0.3",
  157. "lxml==4.9.1",
  158. "sasl==0.3.1",
  159. "Markdown==3.8",
  160. ],
  161. "3.11": [
  162. "cryptography==42.0.8",
  163. "numpy==1.24.4",
  164. "pandas==2.0.3",
  165. "lxml==4.9.1",
  166. "async-timeout==5.0.1",
  167. "pure-sasl==0.6.2",
  168. "Markdown==3.8"
  169. ],
  170. }
  171. self.pytorch_requirements = {
  172. "default": [
  173. "--extra-index-url \"https://cloudera-build-us-west-1.vpc.cloudera.com/whl/cpu\" torch==2.2.2+cpu torchvision==0.17.2+cpu",
  174. ]
  175. }
  176. self.arch_requirements_map = {
  177. "ppc64le": self.ppc64le_requirements,
  178. "x86_64": self.x86_64_requirements,
  179. "aarch64": self.aarch64_requirements,
  180. }
  181. self.arch = platform.machine()
  182. self.python_version_string = f"{sys.version_info.major}.{sys.version_info.minor}"
  183. def copy_local_requirements(self, python_version_string):
  184. local_dir = f"{this_dir}/{python_version_string}"
  185. if os.path.exists(local_dir):
  186. shutil.rmtree(local_dir, ignore_errors=True)
  187. shutil.copytree(f"{this_dir}/ext-py3", local_dir)
  188. return list(map(lambda x: f"file://{local_dir}/{x}", self.local_requirements))
  189. def generate_requirements(self):
  190. if self.arch not in self.arch_requirements_map:
  191. raise ValueError(f"Unsupported architecture: {self.arch}")
  192. arch_reqs = self.arch_requirements_map[self.arch]
  193. self.requirements.extend(arch_reqs.get(self.python_version_string, arch_reqs["default"]))
  194. self.requirements.extend(self.copy_local_requirements(self.python_version_string))
  195. self.requirements.extend(self.pytorch_requirements.get(self.python_version_string, self.pytorch_requirements["default"]))
  196. with open(f"{this_dir}/requirements-{self.arch}-{self.python_version_string}.txt", "w") as f:
  197. f.write("\n".join(self.requirements))
  198. def get_file_name(self):
  199. return f"{this_dir}/requirements-{self.arch}-{self.python_version_string}.txt"
  200. if __name__ == "__main__":
  201. generator = RequirementsGenerator()
  202. try:
  203. generator.generate_requirements()
  204. print(generator.get_file_name())
  205. except Exception as e:
  206. print(f"Error: {e}")
  207. sys.exit(1)