| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390 |
- #
- # Licensed to Cloudera, Inc. under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. Cloudera, Inc. licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- #
- # <<<< DEV ONLY
- #
- # Hue Build Process Overview
- # =======================================
- # This build process handles different types of builds:
- #
- # - The production release build, the main task of which is to install
- # Desktop in the local system. The main build products are:
- #
- # * build/env - The virtual environment, where Desktop core
- # and various apps install into.
- #
- # - The development build, which includes test and debugging tools. It
- # generates the production release, which boils down to a tarball that is
- # downloaded and installed. Additional build products include:
- #
- # * build/release/prod - The production tarball.
- # * build/docs - General Desktop and SDK docs.
- #
- # We achieve this by selecting the parts to be distributed, and by stripping
- # the Makefile's. Lines enclosed by the "DEV ONLY" marks are stripped for the
- # general release. Targets are split/expanded using multiple rules and
- # double-colon rules. The idea is to use almost identical build logic for the
- # different flavours.
- #
- # Logic Flow
- # ==========
- # Here we summarize the flow of the build logic. Lines represent dependency.
- #
- # virtual-bootstrap.py
- # |
- # virtual-env (./env)
- # |
- # +----------+
- # | |
- # | V
- # | desktop <--- recursive make in /desktop
- # | |
- # | V
- # | apps <--- recursive make in /apps
- # |
- # V
- # prod tarball <-- selective copying into /build/release/...
- #
- #
- # Application Build
- # =================
- # An app typically includes $(ROOT)/Makefile.sdk, which provides the standard
- # targets and facilities. ROOT, which points to the Hue installation root, is
- # always passed in.
- #
- # The application may also choose to not use the Hue build facilities. In that
- # case, its Makefile still receives $(ROOT), and needs to provide several
- # targets as specified in Makefile.sdk.
- #
- # END DEV ONLY >>>>
- ###################################
- # Global variables
- ###################################
- ROOT := $(realpath .)
- PPC64LE := $(shell uname -m)
- include $(ROOT)/Makefile.vars.priv
- ###################################
- # Error checking
- ###################################
- .PHONY: default
- default:
- @echo 'The build targets for Hue $(DESKTOP_VERSION) are:'
- @echo ' install : Install at $$PREFIX ($(INSTALL_DIR)); need desktop'
- @echo ' apps : Build and register all desktop apps; need desktop'
- @echo ' desktop : Build desktop core only'
- @echo ' clean : Remove desktop build products'
- @echo ' distclean : Remove desktop and thirdparty build products'
- # <<<< DEV ONLY
- @echo ' doc : Build documentation'
- @echo ' prod : Generate a tar file for production distribution'
- @echo ' locales : Extract strings and update dictionary of each locale'
- @echo ' ace : Builds the Ace Editor tool'
- # END DEV ONLY >>>>
- .PHONY: all
- all: default
- # <<<< DEV ONLY
- include Makefile.tarball
- ###################################
- # Build docs (unused)
- ###################################
- .PHONY: docs
- docs:
- @$(MAKE) -C docs
- # END DEV ONLY >>>>
- ###################################
- # Install parent POM
- ###################################
- parent-pom:
- ifneq (,$(BUILD_DB_PROXY))
- cd $(ROOT)/maven && mvn install $(MAVEN_OPTIONS)
- endif
- .PHONY: parent-pom
- ###################################
- # virtual-env
- ###################################
- .PHONY: virtual-env
- virtual-env: $(BLD_DIR_ENV)/stamp
- $(BLD_DIR_ENV)/stamp:
- @echo "--- Creating virtual environment at $(BLD_DIR_ENV)"
- ifeq ($(PYTHON_VER),python2.7)
- @$(SYS_PYTHON) $(VIRTUAL_BOOTSTRAP) $(VIRTUALENV_OPTS) --system-site-packages $(BLD_DIR_ENV)
- else ifeq ($(PYTHON_VER),python3.8)
- @$(SYS_PYTHON) -m pip install --upgrade pip==22.2.2
- @$(SYS_PIP) install virtualenv==20.16.5 virtualenv-make-relocatable==0.0.1
- @if [[ "ppc64le" == $(PPC64LE) ]]; then \
- $(SYS_PYTHON) -m venv $(BLD_DIR_ENV); \
- fi
- @virtualenv -p $(PYTHON_VER) $(BLD_DIR_ENV)
- endif
- @echo "--- Virtual environment $(BLD_DIR_ENV) ready"
- @touch $@
- @echo '--- Installing PIP_MODULES in virtual-env'
- ifeq ($(PYTHON_VER),python2.7)
- @echo "--- start installing PIP_MODULES in virtual-env"
- @$(ENV_PIP) install --upgrade pip
- @$(ENV_PIP) install --upgrade --force-reinstall $(PIP_MODULES)
- @echo "--- done installing PIP_MODULES in virtual-env"
- else ifeq ($(PYTHON_VER),python3.8)
- @if [[ "ppc64le" == $(PPC64LE) ]]; then \
- echo '--- Installing $(REQUIREMENT_PPC64LE_FILE) into virtual-env via $(ENV_PIP)'; \
- $(ENV_PIP) install -r $(REQUIREMENT_PPC64LE_FILE); \
- echo '--- Finished $(REQUIREMENT_PPC64LE_FILE) into virtual-env'; \
- else \
- echo '--- Installing $(REQUIREMENT_FILE) into virtual-env via $(ENV_PIP)'; \
- $(ENV_PIP) install -r $(REQUIREMENT_FILE); \
- echo '--- Finished $(REQUIREMENT_FILE) into virtual-env'; \
- fi
- @$(ENV_PIP) install $(NAVOPTAPI_WHL)
- @echo '--- Finished $(NAVOPTAPI_WHL) into virtual-env'
- @touch $(REQUIREMENT_DOT_FILE)
- endif
- ###################################
- # Build desktop
- ###################################
- .PHONY: desktop
- # <<<< DEV ONLY
- desktop: parent-pom
- # END DEV ONLY >>>>
- desktop: virtual-env
- @$(MAKE) -C desktop
- ###################################
- # Build apps
- ###################################
- .PHONY: apps
- apps: desktop
- @$(MAKE) npm-install
- @$(MAKE) -C $(APPS_DIR) env-install
- @$(MAKE) create-static
- ###################################
- # Install binary dist
- ###################################
- INSTALL_CORE_FILES = \
- Makefile* $(wildcard *.mk) \
- ext \
- tools/app_reg \
- tools/virtual-bootstrap \
- tools/enable-python27.sh \
- tools/relocatable.sh \
- VERS* LICENSE* README*
- .PHONY: install
- install: virtual-env install-check install-core-structure install-desktop install-apps install-env
- .PHONY: install-check
- install-check:
- @if [ -n '$(wildcard $(INSTALL_DIR)/*)' ] ; then \
- echo 'ERROR: $(INSTALL_DIR) not empty. Cowardly refusing to continue.' ; \
- false ; \
- fi
- .PHONY: install-core-structure
- install-core-structure:
- @echo --- Installing core source structure in $(INSTALL_DIR)...
- @mkdir -p $(INSTALL_DIR)
- @tar cf - $(INSTALL_CORE_FILES) | tar -C $(INSTALL_DIR) -xf -
- @# Add some variables to Makefile to make sure that our virtualenv
- @# in the install root is the same one we built from - this also
- @# disables the check for python-devel packages which are no longer
- @# needed
- @echo "SYS_PYTHON=$(ENV_PYTHON_VERSION)" > $(INSTALL_DIR)/Makefile.buildvars
- @echo "SKIP_PYTHONDEV_CHECK=1" >> $(INSTALL_DIR)/Makefile.buildvars
- .PHONY: install-desktop
- install-desktop:
- @echo --- Installing Desktop core...
- INSTALL_DIR=$(realpath $(INSTALL_DIR)) $(MAKE) -C desktop install
- .PHONY: install-apps
- install-apps:
- @echo '--- Installing Applications...'
- INSTALL_DIR=$(realpath $(INSTALL_DIR)) $(MAKE) -C apps install
- .PHONY: install-env
- install-env:
- @echo --- Creating new virtual environment
- $(MAKE) -C $(INSTALL_DIR) virtual-env
- @echo --- Setting up Desktop core
- $(MAKE) -C $(INSTALL_DIR)/desktop env-install
- @echo --- Setting up Applications
- $(MAKE) -C $(INSTALL_DIR)/apps env-install
- @echo --- Setting up Frontend assets
- cp $(ROOT)/package.json $(INSTALL_DIR)
- cp $(ROOT)/package-lock.json $(INSTALL_DIR)
- cp $(ROOT)/webpack.config*.js $(INSTALL_DIR)
- cp $(ROOT)/babel.config.js $(INSTALL_DIR)
- cp $(ROOT)/tsconfig.json $(INSTALL_DIR)
- $(MAKE) -C $(INSTALL_DIR) npm-install
- $(MAKE) -C $(INSTALL_DIR) create-static
- ###################################
- # Frontend and static assets
- ###################################
- .PHONY: npm-install
- npm-install:
- npm --version
- node --version
- npm install
- npm run webpack
- npm run webpack-login
- npm run webpack-workers
- node_modules/.bin/removeNPMAbsolutePaths .
- rm -rf node_modules
- .PHONY: create-static
- create-static:
- ./build/env/bin/python ./build/env/bin/hue collectstatic --noinput
- # <<<< DEV ONLY
- .PHONY: doc
- doc:
- hugo --source docs/docs-site
- # END DEV ONLY >>>>
- ###################################
- # Internationalization
- ###################################
- # <<<< DEV ONLY
- .PHONY: locales
- locales:
- @$(MAKE) -C desktop compile-locales
- @$(MAKE) -C apps compile-locales
- # END DEV ONLY >>>>
- ###################################
- # Ace Editor
- ###################################
- # <<<< DEV ONLY
- .PHONY: ace
- ace:
- @cd tools/ace-editor && ./hue-ace.sh
- # END DEV ONLY >>>>
- ###################################
- # JISON Parser Generators
- ###################################
- # <<<< DEV ONLY
- .PHONY: global-search-parser
- global-search-parser:
- @pushd tools/jison/ && npm install && node generateParsers.js globalSearchParser && popd
- .PHONY: solr-all-parsers
- solr-all-parsers:
- @pushd tools/jison/ && npm install && node generateParsers.js solrQueryParser solrFormulaParser && popd
- .PHONY: solr-query-parser
- solr-query-parser:
- @pushd tools/jison/ && npm install && node generateParsers.js solrQueryParser && popd
- .PHONY: solr-formula-parser
- solr-formula-parser:
- @pushd tools/jison/ && npm install && node generateParsers.js solrFormulaParser && popd
- .PHONY: sql-all-parsers
- sql-all-parsers:
- @pushd tools/jison/ && npm install && node generateParsers.js generic hive impala && popd
- .PHONY: sql-autocomplete-parser
- sql-autocomplete-parser:
- @pushd tools/jison/ && npm install && node generateParsers.js genericAutocomp hiveAutocomp impalaAutocomp && popd
- .PHONY: sql-statement-parser
- sql-statement-parser:
- @pushd tools/jison/ && npm install && node generateParsers.js sqlStatementsParser && popd
- .PHONY: sql-syntax-parser
- sql-syntax-parser:
- @pushd tools/jison/ && npm install && node generateParsers.js genericSyntax hiveSyntax impalaSyntax && popd
- # END DEV ONLY >>>>
- ###################################
- # Cleanup
- ###################################
- .PHONY: clean
- clean:
- @$(MAKE) -C desktop clean
- @$(MAKE) -C apps clean
- # <<<< DEV ONLY
- @$(MAKE) -C docs clean
- # END DEV ONLY >>>>
- @rm -rf $(BLD_DIR_ENV)
- @rm -rf $(STATIC_DIR)
- #
- # Note: It is important for clean targets to *ONLY* clean products of the
- # build, and not misc runtime generated files. Don't abuse Makefile.
- #
- .PHONY: distclean
- distclean: clean
- @# Remove the other directories in build/
- @$(MAKE) -C desktop distclean
- @$(MAKE) -C apps distclean
- @rm -rf $(BLD_DIR)
- .PHONY: ext-clean
- ext-clean:
- @$(MAKE) -C desktop ext-clean
- @$(MAKE) -C apps ext-clean
- # <<<< DEV ONLY
- ###############################################
- # Misc (some used by automated test scripts)
- ###############################################
- test:
- DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue test fast --with-xunit
- test-slow:
- DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue test all --with-xunit --with-cover
- $(BLD_DIR_BIN)/coverage xml -i
- start-dev:
- DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue runserver_plus
- devinstall:
- npm run devinstall
- css:
- npm run less
- # END DEV ONLY >>>>
|