Makefile 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428
  1. #
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. #
  18. # <<<< DEV ONLY
  19. #
  20. # Hue Build Process Overview
  21. # =======================================
  22. # This build process handles different types of builds:
  23. #
  24. # - The production release build, the main task of which is to install
  25. # Desktop in the local system. The main build products are:
  26. #
  27. # * build/env - The virtual environment, where Desktop core
  28. # and various apps install into.
  29. #
  30. # - The development build, which includes test and debugging tools. It
  31. # generates the production release, which boils down to a tarball that is
  32. # downloaded and installed. Additional build products include:
  33. #
  34. # * build/release/prod - The production tarball.
  35. # * build/docs - General Desktop and SDK docs.
  36. #
  37. # We achieve this by selecting the parts to be distributed, and by stripping
  38. # the Makefile's. Lines enclosed by the "DEV ONLY" marks are stripped for the
  39. # general release. Targets are split/expanded using multiple rules and
  40. # double-colon rules. The idea is to use almost identical build logic for the
  41. # different flavours.
  42. #
  43. # Logic Flow
  44. # ==========
  45. # Here we summarize the flow of the build logic. Lines represent dependency.
  46. #
  47. # virtual-bootstrap.py
  48. # |
  49. # virtual-env (./env)
  50. # |
  51. # +----------+
  52. # | |
  53. # | V
  54. # | desktop <--- recursive make in /desktop
  55. # | |
  56. # | V
  57. # | apps <--- recursive make in /apps
  58. # |
  59. # V
  60. # prod tarball <-- selective copying into /build/release/...
  61. #
  62. #
  63. # Application Build
  64. # =================
  65. # An app typically includes $(ROOT)/Makefile.sdk, which provides the standard
  66. # targets and facilities. ROOT, which points to the Hue installation root, is
  67. # always passed in.
  68. #
  69. # The application may also choose to not use the Hue build facilities. In that
  70. # case, its Makefile still receives $(ROOT), and needs to provide several
  71. # targets as specified in Makefile.sdk.
  72. #
  73. # END DEV ONLY >>>>
  74. ###################################
  75. # Global variables
  76. ###################################
  77. ROOT := $(realpath .)
  78. PPC64LE := $(shell uname -m)
  79. VIRTUAL_ENV_VERSION := 20.24.4
  80. VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
  81. include $(ROOT)/Makefile.vars.priv
  82. ###################################
  83. # Error checking
  84. ###################################
  85. .PHONY: default
  86. default:
  87. @echo 'The build targets for Hue $(DESKTOP_VERSION) are:'
  88. @echo ' install : Install at $$PREFIX ($(INSTALL_DIR)); need desktop'
  89. @echo ' apps : Build and register all desktop apps; need desktop'
  90. @echo ' desktop : Build desktop core only'
  91. @echo ' clean : Remove desktop build products'
  92. @echo ' distclean : Remove desktop and thirdparty build products'
  93. # <<<< DEV ONLY
  94. @echo ' doc : Build documentation'
  95. @echo ' prod : Generate a tar file for production distribution'
  96. @echo ' locales : Extract strings and update dictionary of each locale'
  97. @echo ' ace : Builds the Ace Editor tool'
  98. # END DEV ONLY >>>>
  99. .PHONY: all
  100. all: default
  101. # <<<< DEV ONLY
  102. include Makefile.tarball
  103. ###################################
  104. # Build docs (unused)
  105. ###################################
  106. .PHONY: docs
  107. docs:
  108. @$(MAKE) -C docs
  109. # END DEV ONLY >>>>
  110. ###################################
  111. # Install parent POM
  112. ###################################
  113. parent-pom:
  114. ifneq (,$(BUILD_DB_PROXY))
  115. cd $(ROOT)/maven && mvn install $(MAVEN_OPTIONS)
  116. endif
  117. .PHONY: parent-pom
  118. ###################################
  119. # virtual-env
  120. ###################################
  121. .PHONY: virtual-env
  122. virtual-env: virtual-env2.7 virtual-env3.8 virtual-env3.9
  123. .PHONY: virtual-env2.7
  124. virtual-env2.7: $(BLD_DIR_ENV)/stamp
  125. $(BLD_DIR_ENV)/stamp:
  126. ifeq ($(PYTHON_VER),python2.7)
  127. @echo "--- Creating virtual environment at $(BLD_DIR_ENV) using $(PYTHON_VER)"
  128. @$(SYS_PYTHON) $(VIRTUAL_BOOTSTRAP) $(VIRTUALENV_OPTS) --system-site-packages $(BLD_DIR_ENV)
  129. @echo "--- Virtual environment $(BLD_DIR_ENV) ready"
  130. @touch $@
  131. @echo '--- Installing PIP_MODULES in virtual-env'
  132. @echo "--- start installing PIP_MODULES in virtual-env"
  133. @$(ENV_PIP) install --upgrade pip
  134. @$(ENV_PIP) install --upgrade --force-reinstall $(PIP_MODULES)
  135. @echo "--- done installing PIP_MODULES in virtual-env"
  136. endif
  137. .PHONY: virtual-env3.8
  138. virtual-env3.8: $(BLD_DIR_ENV)/stamp
  139. $(BLD_DIR_ENV)/stamp:
  140. ifeq ($(PYTHON_VER),python3.8)
  141. @echo "--- Creating virtual environment at $(BLD_DIR_ENV) using $(PYTHON_VER)"
  142. @$(SYS_PYTHON) -m pip install --upgrade pip==22.2.2
  143. @$(SYS_PIP) install virtualenv==$(VIRTUAL_ENV_VERSION) virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
  144. @if [[ "ppc64le" == $(PPC64LE) ]]; then \
  145. $(SYS_PYTHON) -m venv $(BLD_DIR_ENV); \
  146. fi
  147. @virtualenv -p $(PYTHON_VER) $(BLD_DIR_ENV)
  148. @echo "--- Virtual environment $(BLD_DIR_ENV) ready"
  149. @touch $@
  150. @echo '--- Installing PIP_MODULES in virtual-env'
  151. @if [[ "ppc64le" == $(PPC64LE) ]]; then \
  152. echo '--- Installing $(REQUIREMENT_PPC64LE_FILE) into virtual-env via $(ENV_PIP)'; \
  153. $(ENV_PIP) install -r $(REQUIREMENT_PPC64LE_FILE); \
  154. echo '--- Finished $(REQUIREMENT_PPC64LE_FILE) into virtual-env'; \
  155. else \
  156. echo '--- Installing $(REQUIREMENT_FILE) into virtual-env via $(ENV_PIP)'; \
  157. $(ENV_PIP) install -r $(REQUIREMENT_FILE); \
  158. echo '--- Finished $(REQUIREMENT_FILE) into virtual-env'; \
  159. fi
  160. @$(ENV_PIP) install $(NAVOPTAPI_WHL)
  161. @echo '--- Finished $(NAVOPTAPI_WHL) into virtual-env'
  162. @touch $(REQUIREMENT_DOT_FILE)
  163. endif
  164. .PHONY: virtual-env3.9
  165. virtual-env3.9: $(BLD_DIR_ENV)/stamp
  166. $(BLD_DIR_ENV)/stamp:
  167. ifeq ($(PYTHON_VER),python3.9)
  168. @echo "--- Creating virtual environment at $(BLD_DIR_ENV) using $(PYTHON_VER)"
  169. @$(SYS_PYTHON) -m pip install --upgrade pip==22.2.2
  170. @$(SYS_PIP) install virtualenv==20.19.0 virtualenv-make-relocatable==0.0.1
  171. @if [[ "ppc64le" == $(PPC64LE) ]]; then \
  172. $(SYS_PYTHON) -m venv $(BLD_DIR_ENV); \
  173. fi
  174. @virtualenv -p $(PYTHON_VER) $(BLD_DIR_ENV)
  175. @echo "--- Virtual environment $(BLD_DIR_ENV) ready"
  176. @touch $@
  177. @echo '--- Installing PIP_MODULES in virtual-env'
  178. @if [[ "ppc64le" == $(PPC64LE) ]]; then \
  179. echo '--- Installing $(REQUIREMENT_PPC64LE_FILE_PY39) into virtual-env via $(ENV_PIP)'; \
  180. $(ENV_PIP) install -r $(REQUIREMENT_PPC64LE_FILE_PY39); \
  181. echo '--- Finished $(REQUIREMENT_PPC64LE_FILE_PY39) into virtual-env'; \
  182. else \
  183. echo '--- Installing $(REQUIREMENT_FILE) into virtual-env via $(ENV_PIP)'; \
  184. $(ENV_PIP) install -r $(REQUIREMENT_FILE); \
  185. echo '--- Finished $(REQUIREMENT_FILE) into virtual-env'; \
  186. fi
  187. @$(ENV_PIP) install $(NAVOPTAPI_WHL)
  188. @echo '--- Finished $(NAVOPTAPI_WHL) into virtual-env'
  189. @touch $(REQUIREMENT_DOT_FILE)
  190. endif
  191. ###################################
  192. # Build desktop
  193. ###################################
  194. .PHONY: desktop
  195. # <<<< DEV ONLY
  196. desktop: parent-pom
  197. # END DEV ONLY >>>>
  198. desktop: virtual-env
  199. @$(MAKE) -C desktop
  200. ###################################
  201. # Build apps
  202. ###################################
  203. .PHONY: apps
  204. apps: desktop
  205. @$(MAKE) npm-install
  206. @$(MAKE) -C $(APPS_DIR) env-install
  207. @$(MAKE) create-static
  208. ###################################
  209. # Install binary dist
  210. ###################################
  211. INSTALL_CORE_FILES = \
  212. Makefile* $(wildcard *.mk) \
  213. ext \
  214. tools/app_reg \
  215. tools/virtual-bootstrap \
  216. tools/enable-python27.sh \
  217. tools/relocatable.sh \
  218. VERS* LICENSE* README*
  219. .PHONY: install
  220. install: virtual-env install-check install-core-structure install-desktop install-apps install-env
  221. .PHONY: install-check
  222. install-check:
  223. @if [ -n '$(wildcard $(INSTALL_DIR)/*)' ] ; then \
  224. echo 'ERROR: $(INSTALL_DIR) not empty. Cowardly refusing to continue.' ; \
  225. false ; \
  226. fi
  227. .PHONY: install-core-structure
  228. install-core-structure:
  229. @echo --- Installing core source structure in $(INSTALL_DIR)...
  230. @mkdir -p $(INSTALL_DIR)
  231. @tar cf - $(INSTALL_CORE_FILES) | tar -C $(INSTALL_DIR) -xf -
  232. @# Add some variables to Makefile to make sure that our virtualenv
  233. @# in the install root is the same one we built from - this also
  234. @# disables the check for python-devel packages which are no longer
  235. @# needed
  236. @echo "SYS_PYTHON=$(ENV_PYTHON_VERSION)" > $(INSTALL_DIR)/Makefile.buildvars
  237. @echo "SKIP_PYTHONDEV_CHECK=1" >> $(INSTALL_DIR)/Makefile.buildvars
  238. .PHONY: install-desktop
  239. install-desktop:
  240. @echo --- Installing Desktop core...
  241. INSTALL_DIR=$(realpath $(INSTALL_DIR)) $(MAKE) -C desktop install
  242. .PHONY: install-apps
  243. install-apps:
  244. @echo '--- Installing Applications...'
  245. INSTALL_DIR=$(realpath $(INSTALL_DIR)) $(MAKE) -C apps install
  246. .PHONY: install-env
  247. install-env:
  248. @echo --- Creating new virtual environment
  249. $(MAKE) -C $(INSTALL_DIR) virtual-env
  250. @echo --- Setting up Desktop core
  251. $(MAKE) -C $(INSTALL_DIR)/desktop env-install
  252. @echo --- Setting up Applications
  253. $(MAKE) -C $(INSTALL_DIR)/apps env-install
  254. @echo --- Setting up Frontend assets
  255. cp $(ROOT)/package.json $(INSTALL_DIR)
  256. cp $(ROOT)/package-lock.json $(INSTALL_DIR)
  257. cp $(ROOT)/webpack.config*.js $(INSTALL_DIR)
  258. cp $(ROOT)/babel.config.js $(INSTALL_DIR)
  259. cp $(ROOT)/tsconfig.json $(INSTALL_DIR)
  260. $(MAKE) -C $(INSTALL_DIR) npm-install
  261. $(MAKE) -C $(INSTALL_DIR) create-static
  262. ###################################
  263. # Frontend and static assets
  264. ###################################
  265. .PHONY: npm-install
  266. npm-install:
  267. npm --version
  268. node --version
  269. npm install
  270. npm run webpack
  271. npm run webpack-login
  272. npm run webpack-workers
  273. node_modules/.bin/removeNPMAbsolutePaths .
  274. rm -rf node_modules
  275. .PHONY: create-static
  276. create-static:
  277. ./build/env/bin/python ./build/env/bin/hue collectstatic --noinput
  278. # <<<< DEV ONLY
  279. .PHONY: doc
  280. doc:
  281. hugo --source docs/docs-site
  282. # END DEV ONLY >>>>
  283. ###################################
  284. # Internationalization
  285. ###################################
  286. # <<<< DEV ONLY
  287. .PHONY: locales
  288. locales:
  289. @$(MAKE) -C desktop compile-locales
  290. @$(MAKE) -C apps compile-locales
  291. # END DEV ONLY >>>>
  292. ###################################
  293. # Ace Editor
  294. ###################################
  295. # <<<< DEV ONLY
  296. .PHONY: ace
  297. ace:
  298. @cd tools/ace-editor && ./hue-ace.sh
  299. # END DEV ONLY >>>>
  300. ###################################
  301. # JISON Parser Generators
  302. ###################################
  303. # <<<< DEV ONLY
  304. .PHONY: global-search-parser
  305. global-search-parser:
  306. @pushd tools/jison/ && npm install && node generateParsers.js globalSearchParser && popd
  307. .PHONY: solr-all-parsers
  308. solr-all-parsers:
  309. @pushd tools/jison/ && npm install && node generateParsers.js solrQueryParser solrFormulaParser && popd
  310. .PHONY: solr-query-parser
  311. solr-query-parser:
  312. @pushd tools/jison/ && npm install && node generateParsers.js solrQueryParser && popd
  313. .PHONY: solr-formula-parser
  314. solr-formula-parser:
  315. @pushd tools/jison/ && npm install && node generateParsers.js solrFormulaParser && popd
  316. .PHONY: sql-all-parsers
  317. sql-all-parsers:
  318. @pushd tools/jison/ && npm install && node generateParsers.js generic hive impala && popd
  319. .PHONY: sql-autocomplete-parser
  320. sql-autocomplete-parser:
  321. @pushd tools/jison/ && npm install && node generateParsers.js genericAutocomp hiveAutocomp impalaAutocomp && popd
  322. .PHONY: sql-statement-parser
  323. sql-statement-parser:
  324. @pushd tools/jison/ && npm install && node generateParsers.js sqlStatementsParser && popd
  325. .PHONY: sql-syntax-parser
  326. sql-syntax-parser:
  327. @pushd tools/jison/ && npm install && node generateParsers.js genericSyntax hiveSyntax impalaSyntax && popd
  328. # END DEV ONLY >>>>
  329. ###################################
  330. # Cleanup
  331. ###################################
  332. .PHONY: clean
  333. clean:
  334. @$(MAKE) -C desktop clean
  335. @$(MAKE) -C apps clean
  336. # <<<< DEV ONLY
  337. @$(MAKE) -C docs clean
  338. # END DEV ONLY >>>>
  339. @rm -rf $(BLD_DIR_ENV)
  340. @rm -rf $(STATIC_DIR)
  341. #
  342. # Note: It is important for clean targets to *ONLY* clean products of the
  343. # build, and not misc runtime generated files. Don't abuse Makefile.
  344. #
  345. .PHONY: distclean
  346. distclean: clean
  347. @# Remove the other directories in build/
  348. @$(MAKE) -C desktop distclean
  349. @$(MAKE) -C apps distclean
  350. @rm -rf $(BLD_DIR)
  351. .PHONY: ext-clean
  352. ext-clean:
  353. @$(MAKE) -C desktop ext-clean
  354. @$(MAKE) -C apps ext-clean
  355. # <<<< DEV ONLY
  356. ###############################################
  357. # Misc (some used by automated test scripts)
  358. ###############################################
  359. test:
  360. DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue test fast --with-xunit
  361. test-slow:
  362. DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue test all --with-xunit --with-cover
  363. $(BLD_DIR_BIN)/coverage xml -i
  364. start-dev:
  365. DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue runserver_plus
  366. devinstall:
  367. npm run devinstall
  368. css:
  369. npm run less
  370. # END DEV ONLY >>>>