Makefile 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. #
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. #
  18. # <<<< DEV ONLY
  19. #
  20. # Hue Build Process Overview
  21. # =======================================
  22. # This build process handles different types of builds:
  23. #
  24. # - The production release build, the main task of which is to install
  25. # Desktop in the local system. The main build products are:
  26. #
  27. # * build/env - The virtual environment, where Desktop core
  28. # and various apps install into.
  29. #
  30. # - The development build, which includes test and debugging tools. It
  31. # generates the production release, which boils down to a tarball that is
  32. # downloaded and installed. Additional build products include:
  33. #
  34. # * build/release/prod - The production tarball.
  35. # * build/docs - General Desktop and SDK docs.
  36. #
  37. # We achieve this by selecting the parts to be distributed, and by stripping
  38. # the Makefile's. Lines enclosed by the "DEV ONLY" marks are stripped for the
  39. # general release. Targets are split/expanded using multiple rules and
  40. # double-colon rules. The idea is to use almost identical build logic for the
  41. # different flavours.
  42. #
  43. # Logic Flow
  44. # ==========
  45. # Here we summarize the flow of the build logic. Lines represent dependency.
  46. #
  47. # virtual-bootstrap.py
  48. # |
  49. # virtual-env (./env)
  50. # |
  51. # +----------+
  52. # | |
  53. # | V
  54. # | desktop <--- recursive make in /desktop
  55. # | |
  56. # | V
  57. # | apps <--- recursive make in /apps
  58. # |
  59. # V
  60. # prod tarball <-- selective copying into /build/release/...
  61. #
  62. #
  63. # Application Build
  64. # =================
  65. # An app typically includes $(ROOT)/Makefile.sdk, which provides the standard
  66. # targets and facilities. ROOT, which points to the Hue installation root, is
  67. # always passed in.
  68. #
  69. # The application may also choose to not use the Hue build facilities. In that
  70. # case, its Makefile still receives $(ROOT), and needs to provide several
  71. # targets as specified in Makefile.sdk.
  72. #
  73. # END DEV ONLY >>>>
  74. ###################################
  75. # Global variables
  76. ###################################
  77. ROOT := $(realpath .)
  78. PPC64LE := $(shell uname -m)
  79. include $(ROOT)/Makefile.vars.priv
  80. ###################################
  81. # Error checking
  82. ###################################
  83. .PHONY: default
  84. default:
  85. @echo 'The build targets for Hue $(DESKTOP_VERSION) are:'
  86. @echo ' install : Install at $$PREFIX ($(INSTALL_DIR)); need desktop'
  87. @echo ' apps : Build and register all desktop apps; need desktop'
  88. @echo ' desktop : Build desktop core only'
  89. @echo ' clean : Remove desktop build products'
  90. @echo ' distclean : Remove desktop and thirdparty build products'
  91. # <<<< DEV ONLY
  92. @echo ' doc : Build documentation'
  93. @echo ' prod : Generate a tar file for production distribution'
  94. @echo ' locales : Extract strings and update dictionary of each locale'
  95. @echo ' ace : Builds the Ace Editor tool'
  96. # END DEV ONLY >>>>
  97. .PHONY: all
  98. all: default
  99. # <<<< DEV ONLY
  100. include Makefile.tarball
  101. ###################################
  102. # Build docs (unused)
  103. ###################################
  104. test_prep:
  105. @$(ENV_PYTHON) -m pip install -r $(REQUIREMENT_TEST_FILE)
  106. ###################################
  107. # Build docs (unused)
  108. ###################################
  109. .PHONY: docs
  110. docs:
  111. @$(MAKE) -C docs
  112. # END DEV ONLY >>>>
  113. ###################################
  114. # Install parent POM
  115. ###################################
  116. .PHONY: parent-pom
  117. parent-pom:
  118. ifneq (,$(BUILD_DB_PROXY))
  119. cd $(ROOT)/maven && mvn install $(MAVEN_OPTIONS)
  120. endif
  121. $(info "PYTHON_VER is $(PYTHON_VER)")
  122. $(info "SYS_PYTHON is $(SYS_PYTHON)")
  123. $(info "ENV_PYTHON is $(ENV_PYTHON)")
  124. $(info "SYS_PIP is $(SYS_PIP)")
  125. $(info "ENV_PIP is $(ENV_PIP)")
  126. $(info "PYTHON_INCLUDE_DIR is $(PYTHON_INCLUDE_DIR)")
  127. $(info "PYTHON_H is $(PYTHON_H)")
  128. $(info "PIP_VERSION is $(PIP_VERSION)")
  129. $(info "VIRTUAL_ENV_VERSION is $(VIRTUAL_ENV_VERSION)")
  130. $(info "RELOCATABLE is $(RELOCATABLE)")
  131. $(info "LOCAL_PY_BIN is $(LOCAL_PY_BIN)")
  132. $(info "BLD_DIR_ENV is $(BLD_DIR_ENV)")
  133. $(info "REQUIREMENT_FILE is $(REQUIREMENT_FILE)")
  134. $(info "INSTALL_DIR is $(INSTALL_DIR)")
  135. $(info "INST_DIR_ENV is $(INST_DIR_ENV)")
  136. $(info "PPC64LE is $(PPC64LE)")
  137. ###################################
  138. # virtual-env
  139. # Enhanced to support building and packaging Hue for multiple Python versions.
  140. # Adds per-Python virtual environment creation, improved logging and diagnostics,
  141. # Test push
  142. ###################################
  143. virtual-env: $(BLD_DIR_ENV)/bin/python
  144. $(BLD_DIR_ENV)/bin/python:
  145. @echo "--- Creating virtual environment for $(PYTHON_VER) ---"
  146. @mkdir -p $(BLD_DIR_ENV)
  147. @$(SYS_PYTHON) -m pip install --upgrade pip==$(PIP_VERSION)
  148. @$(SYS_PYTHON) -m pip install virtualenv==$(VIRTUAL_ENV_VERSION) virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
  149. @$(SYS_PYTHON) -m virtualenv --system-site-packages --copies -p $(PYTHON_VER) $(BLD_DIR_ENV)
  150. @$(ENV_PYTHON) -m pip install virtualenv==$(VIRTUAL_ENV_VERSION) virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
  151. @$(eval RELOCATABLE := $(shell which virtualenv-make-relocatable))
  152. @echo "REQUIREMENT_FILE is $(REQUIREMENT_FILE)"
  153. @$(ENV_PIP) install -r $(REQUIREMENT_FILE)
  154. @echo "--- Virtual environment setup complete for $(PYTHON_VER) ---"
  155. @$(ENV_PIP) install $(NAVOPTAPI_WHL)
  156. @echo "--- Finished installing $(NAVOPTAPI_WHL) into virtual-env ---"
  157. ###################################
  158. # Build desktop
  159. ###################################
  160. .PHONY: desktop
  161. # <<<< DEV ONLY
  162. desktop: parent-pom
  163. # END DEV ONLY >>>>
  164. desktop: virtual-env
  165. @$(MAKE) -C desktop
  166. ###################################
  167. # relocatable-env
  168. # Enhanced to support relocatable .pth files.
  169. ###################################
  170. relocatable-env:
  171. @$(MAKE) virtual-env
  172. @echo "--- Running $(SYS_PYTHON) $(RELOCATABLE) $(BLD_DIR_ENV)"
  173. @$(ENV_PYTHON) $(RELOCATABLE) $(BLD_DIR_ENV)
  174. @echo "--- Setting up pth files $(ENV_PYTHON) $(ROOT)/tools/relocatable.py"
  175. @$(ENV_PYTHON) $(ROOT)/tools/relocatable.py
  176. ###################################
  177. # Build apps
  178. ###################################
  179. .PHONY: apps
  180. apps: desktop
  181. @$(MAKE) npm-install
  182. @$(MAKE) -C $(APPS_DIR) env-install
  183. @$(MAKE) create-static
  184. ###################################
  185. # Install binary dist
  186. ###################################
  187. INSTALL_CORE_FILES = \
  188. Makefile* $(wildcard *.mk) \
  189. ext \
  190. tools/app_reg \
  191. $(INSTALL_DIR) \
  192. VERS* LICENSE* README*
  193. .PHONY: install
  194. install: virtual-env install-check install-core-structure install-desktop install-apps install-env
  195. .PHONY: install-check
  196. install-check:
  197. @if [ -n '$(wildcard $(INST_DIR_ENV)/*)' ] ; then \
  198. echo "ERROR: $(INST_DIR_ENV) is not empty. Contents:" ; \
  199. ls -la "$(INST_DIR_ENV)" ; \
  200. echo 'ERROR: $(INST_DIR_ENV) not empty. Cowardly refusing to continue.' ; \
  201. false ; \
  202. fi
  203. .PHONY: install-core-structure
  204. install-core-structure:
  205. @echo --- Installing core source structure in $(INSTALL_DIR)...
  206. @mkdir -p $(INSTALL_DIR)
  207. @tar cf - $(INSTALL_CORE_FILES) | tar -C $(INSTALL_DIR) -xf -
  208. @# Add some variables to Makefile to make sure that our virtualenv
  209. @# in the install root is the same one we built from - this also
  210. @# disables the check for python-devel packages which are no longer
  211. @# needed
  212. @echo "SYS_PYTHON=$(ENV_PYTHON_VERSION)" > $(INSTALL_DIR)/Makefile.buildvars
  213. @echo "SKIP_PYTHONDEV_CHECK=1" >> $(INSTALL_DIR)/Makefile.buildvars
  214. .PHONY: install-desktop
  215. install-desktop:
  216. @echo --- Installing Desktop core...
  217. INSTALL_DIR=$(realpath $(INSTALL_DIR)) $(MAKE) -C desktop install
  218. .PHONY: install-apps
  219. install-apps:
  220. @echo '--- Installing Applications...'
  221. INSTALL_DIR=$(realpath $(INSTALL_DIR)) $(MAKE) -C apps install
  222. .PHONY: install-env
  223. install-env:
  224. @echo --- Creating new virtual environment
  225. $(MAKE) -C $(INSTALL_DIR) virtual-env
  226. @echo --- Setting up Desktop core
  227. $(MAKE) -C $(INSTALL_DIR)/desktop env-install
  228. @echo --- Setting up Applications
  229. $(MAKE) -C $(INSTALL_DIR)/apps env-install
  230. @echo --- Setting up Frontend assets
  231. cp $(ROOT)/package.json $(INSTALL_DIR)
  232. cp $(ROOT)/package-lock.json $(INSTALL_DIR)
  233. cp $(ROOT)/webpack.config*.js $(INSTALL_DIR)
  234. cp $(ROOT)/babel.config.js $(INSTALL_DIR)
  235. cp $(ROOT)/tsconfig.json $(INSTALL_DIR)
  236. $(MAKE) -C $(INSTALL_DIR) npm-install
  237. $(MAKE) -C $(INSTALL_DIR) create-static
  238. ###################################
  239. # Frontend and static assets
  240. ###################################
  241. .PHONY: npm-install
  242. npm-install: .npm-install-lock
  243. .npm-install-lock:
  244. npm --version
  245. node --version
  246. npm install
  247. npm run webpack
  248. npm run webpack-login
  249. npm run webpack-workers
  250. node_modules/.bin/removeNPMAbsolutePaths .
  251. rm -rf node_modules
  252. touch .npm-install-lock
  253. .PHONY: create-static
  254. create-static:
  255. $(ENV_PYTHON) $(BLD_DIR_BIN)/hue collectstatic --noinput
  256. # <<<< DEV ONLY
  257. .PHONY: doc
  258. doc:
  259. hugo --source docs/docs-site
  260. # END DEV ONLY >>>>
  261. ###################################
  262. # Internationalization
  263. ###################################
  264. # <<<< DEV ONLY
  265. .PHONY: locales
  266. locales:
  267. @$(MAKE) -C desktop compile-locales
  268. @$(MAKE) -C apps compile-locales
  269. # END DEV ONLY >>>>
  270. ###################################
  271. # Ace Editor
  272. ###################################
  273. # <<<< DEV ONLY
  274. .PHONY: ace
  275. ace:
  276. @cd tools/ace-editor && ./hue-ace.sh
  277. # END DEV ONLY >>>>
  278. ###################################
  279. # JISON Parser Generators
  280. ###################################
  281. # <<<< DEV ONLY
  282. .PHONY: global-search-parser
  283. global-search-parser:
  284. @pushd tools/jison/ && npm install && node generateParsers.js globalSearchParser && popd
  285. .PHONY: solr-all-parsers
  286. solr-all-parsers:
  287. @pushd tools/jison/ && npm install && node generateParsers.js solrQueryParser solrFormulaParser && popd
  288. .PHONY: solr-query-parser
  289. solr-query-parser:
  290. @pushd tools/jison/ && npm install && node generateParsers.js solrQueryParser && popd
  291. .PHONY: solr-formula-parser
  292. solr-formula-parser:
  293. @pushd tools/jison/ && npm install && node generateParsers.js solrFormulaParser && popd
  294. .PHONY: sql-all-parsers
  295. sql-all-parsers:
  296. @pushd tools/jison/ && npm install && node generateParsers.js generic hive impala && popd
  297. .PHONY: sql-autocomplete-parser
  298. sql-autocomplete-parser:
  299. @pushd tools/jison/ && npm install && node generateParsers.js genericAutocomp hiveAutocomp impalaAutocomp && popd
  300. .PHONY: sql-statement-parser
  301. sql-statement-parser:
  302. @pushd tools/jison/ && npm install && node generateParsers.js sqlStatementsParser && popd
  303. .PHONY: sql-syntax-parser
  304. sql-syntax-parser:
  305. @pushd tools/jison/ && npm install && node generateParsers.js genericSyntax hiveSyntax impalaSyntax && popd
  306. # END DEV ONLY >>>>
  307. ###################################
  308. # Cleanup
  309. ###################################
  310. .PHONY: clean
  311. clean:
  312. @$(MAKE) -C desktop clean
  313. @$(MAKE) -C apps clean
  314. # <<<< DEV ONLY
  315. @$(MAKE) -C docs clean
  316. # END DEV ONLY >>>>
  317. @rm -rf $(BLD_DIR_ENV)
  318. @rm -rf $(STATIC_DIR)
  319. #
  320. # Note: It is important for clean targets to *ONLY* clean products of the
  321. # build, and not misc runtime generated files. Don't abuse Makefile.
  322. #
  323. .PHONY: distclean
  324. distclean: clean
  325. @# Remove the other directories in build/
  326. @$(MAKE) -C desktop distclean
  327. @$(MAKE) -C apps distclean
  328. @rm -rf $(BLD_DIR)
  329. .PHONY: ext-clean
  330. ext-clean:
  331. @$(MAKE) -C desktop ext-clean
  332. @$(MAKE) -C apps ext-clean
  333. # <<<< DEV ONLY
  334. ###############################################
  335. # Misc (some used by automated test scripts)
  336. ###############################################
  337. huecheck:
  338. @echo "Checking for release..."
  339. DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue check
  340. test:
  341. DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue test fast --with-xunit
  342. test-slow:
  343. DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue test all --with-xunit --with-cover
  344. $(BLD_DIR_BIN)/coverage xml -i
  345. start-dev:
  346. DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue runserver_plus
  347. devinstall:
  348. npm run devinstall
  349. css:
  350. npm run less
  351. # END DEV ONLY >>>>