Переглянути джерело

Enable Multi-python support extension for Hue (#4106)

* Enable Multi-python support extension for Hue

- Introduced infrastructure for building and packaging Hue against multiple Python versions (3.8, 3.9, 3.10, 3.11).
- Refactored `Makefile`, `Makefile.vars`, and various component-level Makefiles to support per-Python virtual environments under `build/venvs/`.
- Removed obsolete `requirements.txt` and `base_requirements.txt`; replaced with dynamic `generate_requirements.py` that resolves architecture- and Python-version-specific dependencies at build time.
- Updated environment bootstrapping (`virtual-env`) logic:
  * More robust handling of Python/PIP binaries across platforms.
  * Support for `virtualenv-make-relocatable` and `.pth` path rewriting via a new `relocatable.py` tool.
- Enhanced build toolchain for Cloudera:
  * `build_hue_cloudera.sh` and `build_hue_common.sh` now iterate builds over supported Python versions.
  * SQLite3 installation now checks for presence and adds proper bin path.
  * Redundant shell logic simplified; system-level dependency installation made cleaner.
- Modernized PEP 517-compliant editable install (`pip install -e .`) for all Python apps/libs (removed `setup.py develop` usage).
- Build output (`make prod`, `make release`) now clearly logs artifact paths and validates packaging exclusions.
- Added `huecheck` target for post-build diagnostics.

* fixing as per Ying's comment

* fixing as per Ayush's comment

* fixing circleci, commitflow and pr-comments for multi-python build

* test commit

* trying out new image for circleci

uncommenting the pytorch_requirements line in generate_requirements.py

* [Pytest] Excluding the appropriate ext-py3 library paths from unittest discovery and using the correct path to run hue

(cherry picked from commit 6f9496ecf5b9101ae69ac7e6fd2faee3491e919e)

---------

Co-authored-by: agl29 <ayush.goyal@cloudera.com>
Prakash Ranade 7 місяців тому
батько
коміт
de00238a35

+ 8 - 8
.circleci/config.yml

@@ -21,15 +21,15 @@ commands:
           command: |
             sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime
             export DEBIAN_FRONTEND=noninteractive
-            sudo apt-get update
-            sudo apt-get install -y gcc g++ build-essential python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev  libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev
+            sudo apt-get update -y
+            sudo apt-get install -y gcc g++ build-essential git sudo curl
+            sudo apt-get install -y curl python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev  libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
             curl -sL https://deb.nodesource.com/setup_20.x | sudo bash - && sudo apt-get install -y nodejs
+            unset PIP_FIND_LINKS
+            unset PIP_EXTRA_INDEX_URL
             curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python3.8
-            sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
 
-            export PYTHON_VER=python3.8
-            export ROOT=$PWD
-            make test_prep
+            PYTHON_VER=python3.8 make apps docs test_prep
 
       - run:
           name: run documentation lints
@@ -52,12 +52,12 @@ commands:
       - run:
           name: run python lints
           command: |
-            PYTHONWARNINGS=always ./build/env/bin/hue runruff check
+            PYTHONWARNINGS=always ./build/venvs/python3.8/bin/hue runruff check
 
       - run:
           name: run tests
           command: |
-            PYTHONWARNINGS=always ./build/env/bin/pytest 
+            PYTHONWARNINGS=always ./build/venvs/python3.8/bin/pytest 
 
       - store_artifacts:
           path: test-reports

+ 11 - 3
.github/workflows/commitflow-py3.yml

@@ -57,15 +57,23 @@ jobs:
 
         export PYTHON_VER=python${{ matrix.python-version }}
         export ROOT=$PWD
-        make test_prep
+        make apps test_prep
 
     - name: Run python unit tests
       run: |
-        ./build/env/bin/pytest
+        if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
+          ./build/env/bin/pytest
+        else
+          ./build/venvs/python${{ matrix.python-version }}/bin/pytest
+        fi
 
     - name: Run python lints
       run: |
-        PYTHONWARNINGS=always ./build/env/bin/hue runruff check
+        if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
+          PYTHONWARNINGS=always ./build/env/bin/hue runruff check
+        else
+          PYTHONWARNINGS=always ./build/venvs/python${{ matrix.python-version }}/bin/hue runruff check
+        fi
 
     - name: Run documentation lints
       run: |

+ 1 - 1
.github/workflows/pr-comments.yml

@@ -46,7 +46,7 @@ jobs:
 
           export PYTHON_VER=python3.11
           export ROOT=$PWD
-          make test_prep
+          make apps test_prep
 
       - name: Run python unit tests
         run: |

+ 56 - 36
Makefile

@@ -96,7 +96,7 @@ default:
 	@echo '  clean       : Remove desktop build products'
 	@echo '  distclean   : Remove desktop and thirdparty build products'
 # <<<< DEV ONLY
-	@echo '  doc 	      : Build documentation'
+	@echo '  doc 	     : Build documentation'
 	@echo '  prod        : Generate a tar file for production distribution'
 	@echo '  locales     : Extract strings and update dictionary of each locale'
 	@echo '  ace         : Builds the Ace Editor tool'
@@ -111,9 +111,8 @@ include Makefile.tarball
 ###################################
 # Build docs (unused)
 ###################################
-.PHONY: test_prep
-test_prep: apps
-	@$(ENV_PIP) install -r $(REQUIREMENT_TEST_FILE)
+test_prep:
+	@$(ENV_PYTHON) -m pip install -r $(REQUIREMENT_TEST_FILE)
 
 ###################################
 # Build docs (unused)
@@ -126,42 +125,46 @@ docs:
 ###################################
 # Install parent POM
 ###################################
+.PHONY: parent-pom
 parent-pom:
 ifneq (,$(BUILD_DB_PROXY))
 	cd $(ROOT)/maven && mvn install $(MAVEN_OPTIONS)
 endif
 
-.PHONY: parent-pom
-
+$(info "PYTHON_VER is $(PYTHON_VER)")
+$(info "SYS_PYTHON is $(SYS_PYTHON)")
+$(info "ENV_PYTHON is $(ENV_PYTHON)")
+$(info "SYS_PIP is $(SYS_PIP)")
+$(info "ENV_PIP is $(ENV_PIP)")
+$(info "PYTHON_INCLUDE_DIR is $(PYTHON_INCLUDE_DIR)")
+$(info "PYTHON_H is $(PYTHON_H)")
+$(info "PIP_VERSION is $(PIP_VERSION)")
+$(info "VIRTUAL_ENV_VERSION is $(VIRTUAL_ENV_VERSION)")
+$(info "RELOCATABLE is $(RELOCATABLE)")
+$(info "LOCAL_PY_BIN is $(LOCAL_PY_BIN)")
+$(info "BLD_DIR_ENV is $(BLD_DIR_ENV)")
+$(info "REQUIREMENT_FILE is $(REQUIREMENT_FILE)")
+$(info "INSTALL_DIR is $(INSTALL_DIR)")
+$(info "INST_DIR_ENV is $(INST_DIR_ENV)")
+$(info "PPC64LE is $(PPC64LE)")
 ###################################
 # virtual-env
+# Enhanced to support building and packaging Hue for multiple Python versions.
+# Adds per-Python virtual environment creation, improved logging and diagnostics,
+# Test push
 ###################################
-
-.PHONY: virtual-env
-virtual-env: $(BLD_DIR_ENV)/stamp
-$(BLD_DIR_ENV)/stamp:
-	@echo "--- Creating virtual environment at $(BLD_DIR_ENV) using $(PYTHON_VER)"
+virtual-env: $(BLD_DIR_ENV)/bin/python
+$(BLD_DIR_ENV)/bin/python:
+	@echo "--- Creating virtual environment for $(PYTHON_VER) ---"
+	@mkdir -p $(BLD_DIR_ENV)
 	@$(SYS_PYTHON) -m pip install --upgrade pip==$(PIP_VERSION)
-	$(SYS_PIP) install virtualenv==$(VIRTUAL_ENV_VERSION) virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
-	@if [[ "ppc64le" == $(PPC64LE) ]]; then \
-	  $(SYS_PYTHON) -m venv $(BLD_DIR_ENV); \
-	 fi
-	@virtualenv -p $(PYTHON_VER) $(BLD_DIR_ENV)
-	@echo "--- Virtual environment $(BLD_DIR_ENV) ready"
-	@touch $@
-	@echo '--- Installing PIP_MODULES in virtual-env'
-	@if [[ "ppc64le" == $(PPC64LE) ]]; then \
-	  echo '--- Installing $(REQUIREMENT_PPC64LE_FILE) into virtual-env via $(ENV_PIP)'; \
-	  $(ENV_PIP) install -r $(REQUIREMENT_PPC64LE_FILE); \
-	  echo '--- Finished $(REQUIREMENT_PPC64LE_FILE) into virtual-env'; \
-	 else \
-	  echo '--- Installing $(REQUIREMENT_FILE) into virtual-env via $(ENV_PIP)'; \
-	  $(ENV_PIP) install -r $(REQUIREMENT_FILE); \
-	  echo '--- Finished $(REQUIREMENT_FILE) into virtual-env'; \
-         fi
+	@$(SYS_PYTHON) -m pip install virtualenv==$(VIRTUAL_ENV_VERSION) virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
+	@$(SYS_PYTHON) -m virtualenv -p $(PYTHON_VER) $(BLD_DIR_ENV) --copies
+	@echo "REQUIREMENT_FILE is $(REQUIREMENT_FILE)"
+	@unset PIP_FIND_LINKS && unset PIP_EXTRA_INDEX_URL && $(ENV_PIP) install -r $(REQUIREMENT_FILE)
+	@echo "--- Virtual environment setup complete for $(PYTHON_VER) ---"
 	@$(ENV_PIP) install $(NAVOPTAPI_WHL)
-	@echo '--- Finished $(NAVOPTAPI_WHL) into virtual-env'
-	@touch $(REQUIREMENT_DOT_FILE)
+	@echo "--- Finished installing $(NAVOPTAPI_WHL) into virtual-env ---"
 ###################################
 # Build desktop
 ###################################
@@ -173,6 +176,16 @@ desktop: parent-pom
 desktop: virtual-env
 	@$(MAKE) -C desktop
 
+###################################
+# relocatable-env
+# Enhanced to support relocatable .pth files.
+###################################
+relocatable-env:
+	@$(MAKE) virtual-env
+	@echo "--- Running $(SYS_PYTHON) $(RELOCATABLE) $(BLD_DIR_ENV)"
+	@$(SYS_PYTHON) $(RELOCATABLE) $(BLD_DIR_ENV)
+	@echo "--- Setting up pth files $(ENV_PYTHON) $(ROOT)/tools/relocatable.py"
+	@$(ENV_PYTHON) $(ROOT)/tools/relocatable.py
 
 ###################################
 # Build apps
@@ -190,8 +203,7 @@ INSTALL_CORE_FILES = \
 	Makefile* $(wildcard *.mk) \
 	ext \
 	tools/app_reg \
-	tools/virtual-bootstrap \
-	tools/relocatable.sh \
+	$(INSTALL_DIR) \
 	VERS* LICENSE* README*
 
 .PHONY: install
@@ -199,8 +211,10 @@ install: virtual-env install-check install-core-structure install-desktop instal
 
 .PHONY: install-check
 install-check:
-	@if [ -n '$(wildcard $(INSTALL_DIR)/*)' ] ; then \
-	  echo 'ERROR: $(INSTALL_DIR) not empty. Cowardly refusing to continue.' ; \
+	@if [ -n '$(wildcard $(INST_DIR_ENV)/*)' ] ; then \
+	  echo "ERROR: $(INST_DIR_ENV) is not empty. Contents:" ; \
+	  ls -la "$(INST_DIR_ENV)" ; \
+	  echo 'ERROR: $(INST_DIR_ENV) not empty. Cowardly refusing to continue.' ; \
 	  false ; \
 	fi
 
@@ -249,7 +263,8 @@ install-env:
 ###################################
 
 .PHONY: npm-install
-npm-install:
+npm-install: .npm-install-lock
+.npm-install-lock:
 	npm --version
 	node --version
 	npm install
@@ -258,10 +273,11 @@ npm-install:
 	npm run webpack-workers
 	node_modules/.bin/removeNPMAbsolutePaths .
 	rm -rf node_modules
+	touch .npm-install-lock
 
 .PHONY: create-static
 create-static:
-	./build/env/bin/python ./build/env/bin/hue collectstatic --noinput
+	$(ENV_PYTHON) $(BLD_DIR_BIN)/hue collectstatic --noinput
 
 # <<<< DEV ONLY
 .PHONY: doc
@@ -366,6 +382,10 @@ ext-clean:
 # Misc (some used by automated test scripts)
 ###############################################
 
+huecheck:
+	@echo "Checking for release..."
+	DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue check
+
 test:
 	DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue test fast --with-xunit
 

+ 5 - 5
Makefile.sdk

@@ -114,8 +114,7 @@ $(EXT_PY_ENV_INSTALL_TARGETS): CUR_EXT_PY=$(notdir $(@D))
 #   	Pip install any 3rd party package that are based on requirements.txt
 #
 .PHONY: ext-env-pip-install
-ext-env-pip-install: $(REQUIREMENT_DOT_FILE)
-$(REQUIREMENT_DOT_FILE):
+ext-env-pip-install:
 	@echo '--- Installing $(REQUIREMENT_FILE) into virtual-env via $(ENV_PIP)'
 	@$(ENV_PIP) install -r $(REQUIREMENT_FILE)
 	@echo '--- Finished $(REQUIREMENT_FILE) into virtual-env'
@@ -222,7 +221,7 @@ sdist:
 	@mkdir -p $(SDIST_DIR)
 	@# Copy sources of our app
 	@rsync $(RSYNC_OPT) ./ $(SDIST_DIR)/ $(SDIST_EXCLUDES)
-	@$(MAKE) -C $(SDIST_DIR) clean
+	@PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $(SDIST_DIR) clean
 	@# Also make a tarball
 	@tar -C $(SDIST_DIR)/.. -czf $(SDIST_TGZ) $(APP_FULL_NAME)
 	@echo "--- Created $(SDIST_TGZ)"
@@ -255,8 +254,9 @@ bdist: ext-eggs compile
 install-bdist: bdist
 	@echo "--- Install built distribution for $(APP_NAME) at $(INSTALL_DIR)"
 	@# Check that INSTALL_DIR is empty
-	@if [ -n '$(wildcard $(INSTALL_DIR)/*)' ] ; then \
-	  echo 'ERROR: $(INSTALL_DIR) not empty. Cowardly refusing to continue.' ; \
+	@echo "Prakash $(BDIST_DIR)/ $(INSTALL_DIR)"
+	@if [ -n '$(wildcard $(INST_DIR_ENV)/*)' ] ; then \
+	  echo 'ERROR: $(INST_DIR_ENV) not empty. Cowardly refusing to continue.' ; \
 	  false ; \
 	fi
 	@mkdir -p $(INSTALL_DIR)

+ 40 - 12
Makefile.tarball

@@ -17,7 +17,7 @@
 
 
 # <<<< DEV ONLY
-
+# Extended packaging rules to support Python version-specific virtualenv outputs.
 PROD_INCLUDES := \
 	apps \
 	desktop \
@@ -26,10 +26,15 @@ PROD_INCLUDES := \
 	NOTICE.txt \
 	Makefile* \
 	maven \
+	build \
+	app.reg \
+	desktop/conf \
+	desktop/desktop.db \
 	tools/app_reg \
 	tools/virtual-bootstrap \
-	tools/relocatable.sh \
+	tools/relocatable.py \
 	tools/load-balancer \
+	tools/scripts \
 	VERSION \
 	webpack-stats*.json \
 	package.json \
@@ -39,13 +44,13 @@ PROD_INCLUDES := \
 	tsconfig.json
 
 # Check for things in BDIST_EXCLUDES in various apps
+# Excludes more dev-only content and handles packaging across architectures.
 PROD_EXCLUDES := \
 	apps/beeswax/thrift \
 	apps/hello \
 	apps/jframegallery \
-	build \
-	desktop/conf \
-	desktop/desktop.db \
+	desktop/core/regenerate_thrift.sh \
+	desktop/core/ext-py3 \
 	desktop/devtools.mk \
 	desktop/libs/hadoop/regenerate-thrift.sh \
 	ext/thirdparty/js/manifest.json \
@@ -57,13 +62,10 @@ define remove_devtree_exclusions
 		      -name '.gitignore' -o \
 		      -name '.*~' -o \
 		      -name '.#*' -o \
-		      -name 'build' -o \
+		      -name '.pylintrc' -o \
 		      -name 'logs' -o \
 		      -name 'tags' -o \
 		      -name 'target' -o \
-		      -name 'desktop.db' -o \
-		      -name 'app.reg' -o \
-		      -name '*.py[co]' -o \
 		      -name '.*.sw?' \) -not -path '*/boto/*' -prune -exec rm -rf {} \;
 endef
 
@@ -71,13 +73,39 @@ endef
 ###################################
 # Packaging for production release
 ###################################
+.PHONY: release
+release:
+	@echo "--- Preparing general distribution tree at $@"
+	@rm -rf $(BLD_DIR_PROD) $(BLD_DIR_PROD_TGZ)
+	@mkdir -p $(BLD_DIR_PROD)
+	@tar --exclude="build/release" -chf - $(PROD_INCLUDES) | tar -C $(BLD_DIR_PROD) -xf -
+
+	@echo "---- Removing exclusions"
+	@for i in $(PROD_EXCLUDES) ; do rm -rf $(BLD_DIR_PROD)/$$i ; done
+	@echo "---- Removing *.py[co] files"
+	@find $(BLD_DIR_PROD) -iname \*.py[co]  -exec rm -f {} \;
+	@echo "---- Performing remove_devtree_exclusions"
+	$(call remove_devtree_exclusions,$(BLD_DIR_PROD))
+
+	@echo "---- Copying misc files"
+	@cp $(ROOT)/dist/* $(BLD_DIR_PROD)
+	@mv $(BLD_DIR_PROD)/desktop/conf{.dist,}
+	@cp -r $(BLD_DIR_DOC) $(BLD_DIR_PROD)/docs
+
+	@echo "---- Stripping Makefiles"
+	@$(call STRIP_DEV, $(ROOT)/Makefile, $(BLD_DIR_PROD)/Makefile)
+	@$(call STRIP_DEV, $(ROOT)/Makefile.vars.priv, $(BLD_DIR_PROD)/Makefile.vars.priv)
+	@$(call STRIP_DEV, $(APPS_DIR)/Makefile, $(BLD_DIR_PROD)/apps/Makefile)
+	@$(call STRIP_DEV, $(DESKTOP_DIR)/Makefile, $(BLD_DIR_PROD)/desktop/Makefile)
+	@tar --exclude="build/release" -C $(BLD_DIR_PROD)/.. -chzf $(BLD_DIR_PROD_TGZ) hue-$(DESKTOP_VERSION)
+	@echo "---- $(BLD_DIR_PROD_TGZ)"
 
 # Make a tarball
 .PHONY: prod
 prod: $(BLD_DIR_PROD_TGZ)
 
 $(BLD_DIR_PROD_TGZ): $(BLD_DIR_PROD)
-	@tar -C $(BLD_DIR_PROD)/.. -czf $(BLD_DIR_PROD_TGZ) hue-$(DESKTOP_VERSION)
+	@tar --exclude="python" --exclude="build/release/prod" -C $(BLD_DIR_PROD)/.. -czf $(BLD_DIR_PROD_TGZ) hue-$(DESKTOP_VERSION)
 	@echo "--- Generated $(BLD_DIR_PROD_TGZ)"
 
 .PHONY: $(BLD_DIR_PROD)
@@ -85,7 +113,7 @@ $(BLD_DIR_PROD): apps docs locales
 	@echo "--- Preparing general distribution tree at $@"
 	@rm -rf $@
 	@mkdir -p $@
-	tar -cf - $(PROD_INCLUDES) | tar -C $(BLD_DIR_PROD) -xf -
+	tar --exclude="python" --exclude="build/release/prod" -cf - $(PROD_INCLUDES) | tar -C $(BLD_DIR_PROD) -xf -
 
 	@echo "---- Removing exclusions"
 	@for i in $(PROD_EXCLUDES) ; do rm -rf $(BLD_DIR_PROD)/$$i ; done
@@ -103,6 +131,6 @@ $(BLD_DIR_PROD): apps docs locales
 	@$(call STRIP_DEV, $(DESKTOP_DIR)/Makefile, $(BLD_DIR_PROD)/desktop/Makefile)
 	@echo "---- Removing build directories"
 	find $(BLD_DIR_PROD) -name build -prune -exec rm -rf {} \;
-	@$(MAKE) -C $(BLD_DIR_PROD) ext-clean
+	@PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $(BLD_DIR_PROD) ext-clean
 
 # END DEV ONLY >>>>

+ 60 - 58
Makefile.vars

@@ -39,65 +39,80 @@ endif
 
 VER_ERR_MSG = "Variable PYTHON_VER is $(PYTHON_VER) but it only supports >= python3.8. If not set, defaults to python3.8."
 
-PYTHON_VER ?= python3.8
-PYTHON_EXE = $(shell echo $(PYTHON_VER) | sed "s/\.//")
-SYS_PIP = "pip3.8"
+# Major rewrite to support multi-python environments.
+PYTHON_VERSIONS ?= python3.8 python3.9 python3.10 python3.11
+PYTHON_VER ?= python3.11
+
+ifeq ($(filter $(PYTHON_VER), $(PYTHON_VERSIONS)),)
+  $(error PYTHON_VER ($(PYTHON_VER)) is not in PYTHON_VERSIONS ($(PYTHON_VERSIONS)))
+endif
+
+##############################
+# Location of the virtual environment
+##############################
+BLD_DIR := $(ROOT)/build
+
+# Default case for Python 3.11
+ifeq ($(PYTHON_VER),python3.11)
+  BLD_DIR_ENV := $(BLD_DIR)/env
+else
+  BLD_DIR_ENV := $(BLD_DIR)/venvs/$(PYTHON_VER)
+endif
+
+ifdef INSTALL_DIR
+  INST_DIR := $(INSTALL_DIR)/build
+
+  ifeq ($(PYTHON_VER),python3.11)
+    INST_DIR_ENV := $(INST_DIR)/env
+  else
+    INST_DIR_ENV := $(INST_DIR)/venvs/$(PYTHON_VER)
+  endif
+endif
+
+BLD_DIR_BIN := $(BLD_DIR_ENV)/bin
+
+# Adds detection and configuration for python3.8 through python3.11.
+# Sets up per-version BLD_DIR_ENV and INST_DIR_ENV variables.
+# Introduces version-specific requirement file generation via generate_requirements.py.
+SYS_PYTHON := $(shell IFS=:; for p in $$PATH; do if [ -x "$$p/$(PYTHON_VER)" ]; then echo "$$p/$(PYTHON_VER)"; break; fi; done)
+SYS_PIP := $(shell echo $(SYS_PYTHON) | sed "s/python\([0-9]\+\.[0-9]\+\)$$/pip\1/")
+ENV_PYTHON := $(shell echo $(BLD_DIR_BIN)/$(shell basename $(SYS_PYTHON)))
+ENV_PIP := $(shell echo $(BLD_DIR_BIN)/$(shell basename $(SYS_PIP)))
+PYTHON_INCLUDE_DIR := $(shell $(SYS_PYTHON) -c "import sysconfig; print(sysconfig.get_path('include'))" 2>/dev/null)
+PYTHON_H := $(shell [ -f "$(PYTHON_INCLUDE_DIR)/Python.h" ] && echo "$(PYTHON_INCLUDE_DIR)/Python.h" || echo "")
+PYTHON_VERSION = $(shell $(SYS_PYTHON) -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
 
 $(info "PYTHON_VER is $(PYTHON_VER).")
 
 ifeq ($(shell echo $(PYTHON_VER) | head -c 8),python3.)
-  PYTHON_VER := $(shell echo $(PYTHON_VER) | sed 's/\(python3\.[0-9]*\).*/\1/')
-  MINOR_VER = $(shell echo $(PYTHON_VER) | sed 's/python3\.//')
+  MINOR_VER = $(shell $(SYS_PYTHON) --version | sed -E 's/Python [0-9]+\.([0-9]+).*/\1/')
   ifeq ($(shell test $(MINOR_VER) -lt 8; echo $$?),0)
     $(error "$(VER_ERR_MSG)")
   endif
   EXT_ENV_INSTALL = ext-env-pip-install
-  SYS_PIP := $(shell which pip$(shell echo $(PYTHON_VER) | sed "s/python//g"))
 else
   $(error "$(VER_ERR_MSG)")
 endif
 
-PYTHON_H ?= $(shell ls /usr/include/$(PYTHON_VER)/Python.h 2>/dev/null || ls /usr/local/include/$(PYTHON_VER)/Python.h 2>/dev/null || find /usr/local/$(PYTHON_EXE)/include/$(PYTHON_VER)* -name Python.h 2>/dev/null || find /opt/rh/rh-$(PYTHON_EXE)/root/usr/include/$(PYTHON_VER)* -name Python.h 2>/dev/null || find /Library/Frameworks/Python.framework/Versions/*/include/$(PYTHON_VER)* -name Python.h 2>/dev/null)
 ifndef SKIP_PYTHONDEV_CHECK
   ifeq ($(PYTHON_H),)
     $(error "Error: must have python development packages for $(PYTHON_VER). Could not find Python.h. Please install $(PYTHON_VER)-devel or $(PYTHON_VER)-dev")
   endif
-  SYS_PYTHON ?= $(shell ls /usr/bin/$(PYTHON_VER) 2>/dev/null || ls /usr/local/bin/$(PYTHON_VER) 2>/dev/null || ls /usr/local/$(PYTHON_EXE)/bin/$(PYTHON_VER) 2>/dev/null || ls /opt/rh/$(PYTHON_EXE)/root/usr/bin/$(PYTHON_VER) 2>/dev/null || ls /opt/rh/rh-$(PYTHON_EXE)/root/usr/bin/$(PYTHON_VER) 2>/dev/null || ls /Library/Frameworks/Python.framework/Versions/*/bin/$(PYTHON_VER) 2>/dev/null)
-else
-  SYS_PYTHON ?= $(shell which $(PYTHON_VER))
 endif
 
 ifeq ($(SYS_PYTHON),)
   $(error "Error: Need python version >= 3.8")
-else
-  $(info "SYS_PYTHON is $(SYS_PYTHON).")
 endif
 
 HADOOP_HOME ?= /usr/lib/hadoop
 
-##############################
-# Location of the virtual environment
-##############################
-BLD_DIR := $(ROOT)/build
-BLD_DIR_ENV := $(BLD_DIR)/env
-BLD_DIR_BIN := $(BLD_DIR_ENV)/bin
+
 
 THIRDPARTY_DIR := $(ROOT)/ext/thirdparty
 THIRDPARTY_JS_DIR := $(THIRDPARTY_DIR)/js
 
 STATIC_DIR := $(BLD_DIR)/static
 
-##############################
-# ENV_PYTHON is the Python installed in the virtual environment. App
-# installation should always use the ENV_PYTHON.
-##############################
-ENV_PYTHON := $(BLD_DIR_ENV)/bin/$(notdir $(SYS_PYTHON))
-ENV_PYTHON_VERSION = $(shell $(ENV_PYTHON) -c 'import sys; print ("python%d.%d"% sys.version_info[:2])')
-
-$(info "ENV_PYTHON is $(ENV_PYTHON).")
-$(info "SYS_PIP is $(SYS_PIP).")
-
-
 ##############################
 # ENV_EASY_INSTALL uses the easy_install script installed in the virtual
 # environment. It must be called as an argument to ENV_PYTHON so the
@@ -105,18 +120,9 @@ $(info "SYS_PIP is $(SYS_PIP).")
 # doesn't arise.
 ##############################
 ENV_EASY_INSTALL := $(ENV_PYTHON) $(BLD_DIR_BIN)/easy_install
-ENV_PIP := $(ENV_PYTHON) $(BLD_DIR_BIN)/pip
+$(info "ENV_PYTHON is $(ENV_PYTHON).")
 $(info "ENV_PIP is $(ENV_PIP).")
-
-PIP_MODULES := \
-	cryptography==3.3.2 \
-	future==0.18.2 \
-	lockfile==0.8 \
-	python-daemon==1.5.1 \
-	pytz==2021.1 \
-	filelock==3.0.12 \
-	djangorestframework==3.9.4
-
+$(info "BLD_DIR_BIN is $(BLD_DIR_BIN).")
 
 ##############################
 # This version is substituted through to the tarballs and packages.
@@ -138,30 +144,26 @@ PYBABEL := $(ROOT)/build/env/bin/pybabel
 ##############################
 # Path to files for pip requirements
 ##############################
-REQUIREMENT_FILE := $(ROOT)/desktop/core/requirements.txt
-REQUIREMENT_DOT_FILE := $(ROOT)/desktop/core/.requirements
+REQUIREMENT_FILE := $(shell $(SYS_PYTHON) $(ROOT)/desktop/core/generate_requirements.py)
 REQUIREMENT_TEST_FILE := $(ROOT)/desktop/core/test_requirements.txt
+REQ_DIR := $(dir $(REQUIREMENT_FILE))
+REQ_BASE := $(notdir $(REQUIREMENT_FILE))
+REQ_BASE_NOEXT := $(basename $(REQ_BASE))
 NAVOPTAPI_WHL := $(ROOT)/desktop/core/wheels/navoptapi-1.0.0-py3-none-any.whl
+LOCAL_PY_BASE := $(shell $(SYS_PYTHON) -m site --user-base)
+LOCAL_PY_BIN := $(LOCAL_PY_BASE)/bin
+export PATH := $(PATH):$(LOCAL_PY_BIN)
+RELOCATABLE := $(shell which virtualenv-make-relocatable)
 
 ##############################
 # Set specific versions for some libraries
 ##############################
-ifeq ($(PYTHON_VER),python3.8)
-  PIP_VERSION := 22.2.2
-  VIRTUAL_ENV_VERSION := 20.24.4
-  VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
-  REQUIREMENT_PPC64LE_FILE := $(ROOT)/desktop/core/requirements_ppc64le.txt
-else ifeq ($(PYTHON_VER),python3.9)
-  PIP_VERSION := 22.2.2
+PIP_VERSION := 22.2.2
+VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
+
+# Set VIRTUAL_ENV_VERSION based on PYTHON_VER
+ifeq ($(PYTHON_VER),python3.9)
   VIRTUAL_ENV_VERSION := 20.19.0
-  VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
-  REQUIREMENT_PPC64LE_FILE := $(ROOT)/desktop/core/requirements_ppc64le_py39.txt
-else ifeq ($(PYTHON_VER),python3.10)
-  PIP_VERSION := 22.2.2
-  VIRTUAL_ENV_VERSION := 20.24.4
-  VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
-else ifeq ($(PYTHON_VER),python3.11)
-  PIP_VERSION := 22.2.2
+else
   VIRTUAL_ENV_VERSION := 20.24.4
-  VIRTUAL_ENV_RELOCATABLE_VERSION := 0.0.1
 endif

+ 8 - 6
apps/Makefile

@@ -34,6 +34,8 @@ APPS := $(shell find . -maxdepth 2 -name "src" -type d|cut -d"/" -f2|sort)
 
 ################################################
 # Install all applications into the Desktop environment
+# Updates all recursive targets to pass through PYTHON_VER, ENV_PYTHON, and VIRTUAL_ENV
+# to ensure application builds and installs work per Python version.
 ################################################
 EGG_INFO_TARGETS := $(APPS:%=.recursive-egg-info/%)
 .recursive-egg-info/%:
@@ -41,7 +43,7 @@ EGG_INFO_TARGETS := $(APPS:%=.recursive-egg-info/%)
 
 .PHONY: env-install
 env-install: $(EGG_INFO_TARGETS)
-	$(APP_REG) --install $(APPS) --relative-paths
+	ENV_PYTHON=$(ENV_PYTHON) PYTHON_VER=$(PYTHON_VER) VIRTUAL_ENV=$(BLD_DIR_ENV) $(APP_REG) --install $(APPS) --relative-paths
 
 
 ################################################
@@ -49,7 +51,7 @@ env-install: $(EGG_INFO_TARGETS)
 ################################################
 INSTALL_BDIST_TARGETS := $(APPS:%=.recursive-install-bdist/%)
 .recursive-install-bdist/%: %
-	INSTALL_DIR=$(INSTALL_DIR)/apps/$< $(MAKE) -C $< install-bdist
+	INSTALL_DIR=$(INSTALL_DIR)/apps/$< PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $< install-bdist
 
 .PHONY: install
 install: install-source-parts $(INSTALL_BDIST_TARGETS)
@@ -75,17 +77,17 @@ EXT_CLEAN_TARGETS := $(APPS:%=.recursive-ext-clean/%)
 
 clean: $(CLEAN_TARGETS)
 .recursive-clean/%:
-	@$(MAKE) -C $* clean
+	@PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $* clean
 	@$(APP_REG) --remove $* ||:
 
 distclean: $(DISTCLEAN_TARGETS)
 .recursive-distclean/%:
-	@$(MAKE) -C $* distclean
+	@PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $* distclean
 	@$(APP_REG) --remove $* ||:
 
 ext-clean: $(EXT_CLEAN_TARGETS)
 .recursive-ext-clean/%:
-	@$(MAKE) -C $* ext-clean
+	@PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $* ext-clean
 
 
 ################################################
@@ -112,4 +114,4 @@ I18N_APPS := about \
 COMPILE_LOCALE_TARGETS := $(I18N_APPS:%=.recursive-compile-locales/%)
 compile-locales: $(COMPILE_LOCALE_TARGETS)
 .recursive-compile-locales/%:
-	-@$(MAKE) -C $* compile-locale
+	-@PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $* compile-locale

+ 8 - 5
desktop/Makefile

@@ -100,22 +100,24 @@ $(DESKTOP_DB): $(BLD_DIR_BIN)/hue
 	fi
 
 # Targets that simply recurse into all of the applications
+# Mirrors changes in `apps/Makefile`, forwarding Python environment variables to recursive builds.
+# Ensures desktop-specific build artifacts are correctly tied to the active Python version.
 ENV_INSTALL_TARGETS := $(APPS:%=.recursive-env-install/%)
 $(info "----------------> ENV_INSTALL_TARGETS: $(ENV_INSTALL_TARGETS)")
 .recursive-env-install/%: %
-	$(MAKE) -C $< env-install
+	PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $< env-install
 
 CLEAN_TARGETS := $(APPS:%=.recursive-clean/%)
 .recursive-clean/%: %
-	$(MAKE) -C $< clean
+	PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $< clean
 
 DISTCLEAN_TARGETS := $(APPS:%=.recursive-distclean/%)
 .recursive-distclean/%: %
-	$(MAKE) -C $< distclean
+	PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $< distclean
 
 EXT_CLEAN_TARGETS := $(APPS:%=.recursive-ext-clean/%)
 .recursive-ext-clean/%: %
-	$(MAKE) -C $< ext-clean
+	PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $< ext-clean
 
 .PHONY: env-install
 env-install: $(ENV_INSTALL_TARGETS)
@@ -151,6 +153,7 @@ INSTALL_BDIST_TARGETS := $(APPS:%=.recursive-install-bdist/%)
 .recursive-install-bdist/%: %
 	INSTALL_DIR=$(INSTALL_DIR)/desktop/$< \
 		INSTALL_CONF_DIR=$(INSTALL_DIR)/desktop/conf \
+		PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) \
 		$(MAKE) -C $< install-bdist
 
 .PHONY: install
@@ -183,5 +186,5 @@ install-source-parts:
 COMPILE_LOCALE_TARGETS := $(APPS:%=.recursive-compile-locales/%)
 compile-locales: $(COMPILE_LOCALE_TARGETS)
 .recursive-compile-locales/%:
-	-@$(MAKE) -C $* compile-locale
+	-@PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $* compile-locale
 # END DEV ONLY >>>>

+ 1 - 1
desktop/core/Makefile

@@ -38,4 +38,4 @@ default::
 env-install: $(EXT_ENV_INSTALL) compile
 	@echo '--- Installing $(APP_NAME) into virtual-env'
 	@echo '--- env-install: EXT_ENV_INSTALL: $(EXT_ENV_INSTALL)'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 201 - 0
desktop/core/generate_requirements.py

@@ -0,0 +1,201 @@
+"""
+Dynamic requirements generator for Hue multi-python build system.
+
+Generates architecture- and Python-version-specific requirement files
+at build time. This tool replaces static requirements.txt and supports:
+  - Local editable paths for Hue extensions.
+  - Binary wheels for specific platforms (e.g., ppc64le).
+  - Architecture-aware dependency lists.
+"""
+
+import os
+import sys
+import shutil
+import platform
+
+this_dir = os.path.dirname(os.path.abspath(__file__))
+
+
+class RequirementsGenerator:
+    def __init__(self):
+        self.local_requirements = [
+            "boto-2.49.0",
+            "django-axes-5.13.0",
+            "django-babel",
+            "djangosaml2-0.18.0",
+            "pysaml2-7.3.1",
+            "python-sasl-0.3.1",
+        ]
+
+        self.requirements = [
+            "setuptools==70.0.0",
+            "apache-ranger==0.0.3",
+            "requests-gssapi==1.2.3",
+            "asn1crypto==0.24.0",
+            "avro-python3==1.8.2",
+            "Babel==2.9.1",
+            "celery[redis]==5.4.0",
+            "cffi==1.15.0",
+            "channels==4.0.0",
+            "channels-redis==4.0.0",
+            "configobj==5.0.9",
+            "cx-Oracle==8.3.0",
+            "django-auth-ldap==4.3.0",
+            "Django==4.1.13",
+            "daphne==3.0.2",
+            "django-redis==5.4.0",
+            "django-celery-beat==2.6.0",
+            "django-celery-results==2.5.1",
+            "django-cors-headers==3.13.0",
+            "django-crequest==2018.5.11",
+            "django-debug-panel==0.8.3",
+            "django-debug-toolbar==3.6.0",
+            "django-extensions==3.2.1",
+            "django-ipware==3.0.2",
+            "django_opentracing==1.1.0",
+            "django_prometheus==1.0.15",
+            "django-webpack-loader==1.0.0",
+            "djangomako==1.3.2",
+            "djangorestframework-simplejwt==5.2.1",
+            "djangorestframework==3.14.0",
+            "future==0.18.3",
+            "gcs-oauth2-boto-plugin==3.0",
+            "greenlet==3.1.1",
+            "gunicorn==23.0.0",
+            "ipython==8.12.2",  # Python >= 3.8
+            "jaeger-client==4.3.0",
+            "jdcal==1.0.1",
+            "kazoo==2.8.0",
+            "kerberos==1.3.0",
+            "kubernetes==26.1.0",
+            "Mako==1.2.3",
+            "Markdown==3.7",
+            "openpyxl==3.0.9",
+            "phoenixdb==1.2.1",
+            "prompt-toolkit==3.0.39",
+            "protobuf==3.20.3",
+            "pyarrow==17.0.0",
+            "pyformance==0.3.2",
+            "python-dateutil==2.8.2",
+            "python-daemon==2.2.4",
+            "python-ldap==3.4.3",
+            "python-oauth2==1.1.0",
+            "python-pam==2.0.2",
+            "pytidylib==0.3.2",
+            "pytz==2021.3",
+            "PyJWT==2.4.0",
+            "PyYAML==6.0.1",
+            "requests==2.32.3",
+            "requests-kerberos==0.14.0",
+            "krb5==0.5.1",  # pinned for Sles12, dep of requests-kerberos 0.14.0
+            "rsa==4.7.2",
+            "ruff==0.4.2",
+            "slack-sdk==3.31.0",
+            "SQLAlchemy==1.3.8",
+            "sqlparse==0.5.0",
+            "tablib==0.13.0",
+            "tabulate==0.8.9",
+            "trino==0.329.0",
+            "git+https://github.com/gethue/thrift.git",
+            "thrift-sasl==0.4.3",
+            "django-utils-six==2.0",
+            "six==1.16.0",
+            "psutil==5.8.0",
+            "drf-spectacular[sidecar]==0.27.2",
+        ]
+        self.ppc64le_requirements = {
+            "default": [],
+            "3.8": [
+                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/cryptography-41.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/numpy-1.23.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/pandas-1.4.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/lxml-4.6.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+                "PyYAML==5.4.1",
+                "sasl==0.3.1",
+            ],
+            "3.9": [
+                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/cryptography-41.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/numpy-1.23.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/pandas-1.4.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/lxml-4.6.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+                "PyYAML==6.0.1",
+                "sasl==0.3.1",
+            ]
+        }
+        self.x86_64_requirements = {
+            "default": [
+                "cryptography==42.0.8",
+                "numpy==1.24.4",
+                "pandas==2.0.3",
+                "lxml==4.9.1",
+                "sasl==0.3.1",
+            ],
+            "3.11": [
+                "cryptography==42.0.8",
+                "numpy==1.24.4",
+                "pandas==2.0.3",
+                "lxml==4.9.1",
+                "async-timeout==5.0.1",
+                "pure-sasl==0.6.2",
+            ],
+        }
+        self.aarch64_requirements = {
+            "default": [
+                "cryptography==42.0.8",
+                "numpy==1.24.4",
+                "pandas==2.0.3",
+                "lxml==4.9.1",
+                "sasl==0.3.1",
+            ],
+            "3.11": [
+                "cryptography==42.0.8",
+                "numpy==1.24.4",
+                "pandas==2.0.3",
+                "lxml==4.9.1",
+                "async-timeout==5.0.1",
+                "pure-sasl==0.6.2",
+            ],
+        }
+        self.pytorch_requirements = {
+            "default": [
+                "--extra-index-url \"https://cloudera-build-us-west-1.vpc.cloudera.com/whl/cpu\" torch==2.2.2+cpu torchvision==0.17.2+cpu",
+            ]
+        }
+        self.arch_requirements_map = {
+            "ppc64le": self.ppc64le_requirements,
+            "x86_64": self.x86_64_requirements,
+            "aarch64": self.aarch64_requirements,
+        }
+        self.arch = platform.machine()
+        self.python_version_string = f"{sys.version_info.major}.{sys.version_info.minor}"
+
+    def copy_local_requirements(self, python_version_string):
+        local_dir = f"{this_dir}/{python_version_string}"
+        if os.path.exists(local_dir):
+            shutil.rmtree(local_dir, ignore_errors=True)
+        shutil.copytree(f"{this_dir}/ext-py3", local_dir)
+        return list(map(lambda x: f"file://{local_dir}/{x}", self.local_requirements))
+
+    def generate_requirements(self):
+        if self.arch not in self.arch_requirements_map:
+            raise ValueError(f"Unsupported architecture: {self.arch}")
+        arch_reqs = self.arch_requirements_map[self.arch]
+        self.requirements.extend(arch_reqs.get(self.python_version_string, arch_reqs["default"]))
+        self.requirements.extend(self.copy_local_requirements(self.python_version_string))
+        self.requirements.extend(self.pytorch_requirements.get(self.python_version_string, self.pytorch_requirements["default"]))
+
+        with open(f"{this_dir}/requirements-{self.arch}-{self.python_version_string}.txt", "w") as f:
+            f.write("\n".join(self.requirements))
+
+    def get_file_name(self):
+        return f"{this_dir}/requirements-{self.arch}-{self.python_version_string}.txt"
+
+
+if __name__ == "__main__":
+    generator = RequirementsGenerator()
+    try:
+        generator.generate_requirements()
+        print(generator.get_file_name())
+    except Exception as e:
+        print(f"Error: {e}")
+        sys.exit(1)

+ 0 - 6
desktop/core/requirements.txt

@@ -1,6 +0,0 @@
-cryptography==42.0.8
-numpy==1.24.4
-pandas==2.0.3
-lxml==4.9.1
--r ${ROOT}/desktop/core/base_requirements.txt
-

+ 6 - 1
desktop/core/src/desktop/management/commands/runruff.py

@@ -107,7 +107,12 @@ class Command(BaseCommand):
       *args: Variable arguments.
       **options: Keyword arguments.
     """
-    ruff_package = paths.get_build_dir('env', 'bin', 'ruff')
+    PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
+
+    if PYTHON_VERSION == "3.11":
+        ruff_package = paths.get_build_dir('env', 'bin', 'ruff')
+    else:
+        ruff_package = paths.get_build_dir('venvs', f'python{PYTHON_VERSION}', 'bin', 'ruff')
 
     if not os.path.exists(ruff_package):
       msg = _(

+ 1 - 1
desktop/libs/aws/Makefile

@@ -31,4 +31,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/azure/Makefile

@@ -31,4 +31,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/dashboard/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/hadoop/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/indexer/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/kafka/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/libanalyze/Makefile

@@ -31,4 +31,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/liboauth/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/liboozie/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/librdbms/Makefile

@@ -33,7 +33,7 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .
 
 
 ###################################

+ 1 - 1
desktop/libs/libsaml/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/libsentry/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/libsolr/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/libzookeeper/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/metadata/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
desktop/libs/notebook/Makefile

@@ -33,4 +33,4 @@ default::
 .PHONY: env-install
 env-install: compile ext-env-install
 	@echo '--- Installing $(APP_NAME) into virtual-env'
-	@$(ENV_PYTHON) setup.py develop -N -q
+	@$(ENV_PYTHON) -m pip install -e .

+ 1 - 1
pyproject.toml

@@ -14,7 +14,7 @@ addopts = """
 --cov-report html:reports/code-cov/coverage-html \
 --cov-report xml:reports/code-cov/coverage.xml \
 """
-norecursedirs = "desktop/core/ext-py3"
+norecursedirs = "desktop/core/ext-py3 desktop/core/3.11 desktop/core/3.10 desktop/core/3.9 desktop/core/3.8"
 python_files = "tests.py test_*.py *_tests.py tests_* *_test.py"
 markers = [
     "integration: live server based tests",

+ 12 - 7
tools/app_reg/common.py

@@ -15,12 +15,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import glob
-import logging
 import os
 import sys
-from posixpath import curdir, sep, pardir, join
-
+import glob
+import logging
+from posixpath import curdir, join, pardir, sep
 
 # The root of the Hue installation
 INSTALL_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))
@@ -37,16 +36,20 @@ HUE_PTH_DIR = os.environ.get('HUE_PTH_DIR', None)
 # The Hue config directory
 HUE_CONF_DIR = os.path.join(INSTALL_ROOT, 'desktop', 'conf')
 
+# The Python executable in virtualenv
+ENV_PYTHON = os.environ.get("ENV_PYTHON")
+
+PYTHON_VER = os.environ.get("PYTHON_VER", "")
+
 # Virtual env
-VIRTUAL_ENV = os.path.join(INSTALL_ROOT, 'build', 'env')
+VIRTUAL_ENV = os.environ.get("VIRTUAL_ENV")
 
-# The Python executable in virtualenv
-ENV_PYTHON = os.path.join(VIRTUAL_ENV, 'bin', 'python')
 
 def cmp_version(ver1, ver2):
   """Compare two version strings in the form of 1.2.34"""
   return cmp([int(v) for v in ver1.split('.')], [int(v) for v in ver2.split('.')])
 
+
 def _get_python_lib_dir():
   glob_path = os.path.join(VIRTUAL_ENV, 'lib', 'python*')
   res = glob.glob(glob_path)
@@ -58,9 +61,11 @@ def _get_python_lib_dir():
                       "Please `make clean' first." % glob_path)
   return res[0]
 
+
 def _get_python_site_packages_dir():
   return os.path.join(_get_python_lib_dir(), 'site-packages')
 
+
 def cmp(x, y):
   """
   Replacement for built-in function cmp that was removed in Python 3

+ 54 - 41
tools/cloudera/build_hue_cloudera.sh

@@ -48,20 +48,13 @@ function install_prerequisite() {
     redhat7_ppc_install
   fi
 
-  export SQLITE3_PATH=${SQLITE3_PATH:-"$TOOLS_HOME/sqlite/sqlite3"}
-  if [[ $1 == "redhat9" || $1 == "redhat9_ppc" || $1 == "redhat8" || $1 == "redhat8-arm64" ]]; then
-    check_python39_path
-  elif [[ $1 == "ubuntu22" || $1 == "sles15" ]]; then
-    check_python310_path
-  else
-    check_python38_path
-  fi
-  check_sqlite3
+  export SQLITE3_PATH=${SQLITE3_PATH:-"$TOOLS_HOME/sqlite/bin/sqlite3"}
+  # check_sqlite3
   if [[ $1 == "centos7" ]]; then
     export PYTHON38_PATH=/opt/cloudera/cm-agent
     export pip38_bin="$PYTHON38_PATH/bin/pip3.8"
     centos7_install
-  elif [[ $1 == "redhat8" ]]; then
+  elif [[ $1 == "redhat8" || $1 == "snapshot" ]]; then
     redhat8_install
   elif [[ $1 == "sles12" ]]; then
     sles12_install
@@ -93,39 +86,59 @@ HUE_JAR_VERSION=$3
 export VIRTUAL_ENV_VERSION="20.24.4"
 export VIRTUAL_ENV_RELOCATABLE_VERSION="0.0.1"
 
-big_console_header "Hue PreRequisite Start" "$@"
-install_prerequisite $DOCKEROS
-big_console_header "Hue PreRequisite End" "$@"
-
 export DESKTOP_VERSION=$2
 export HUE_WEBPACK_CONFIG='webpack.config.internal.js'
-export PYTHON_H=$PYTHON38_PATH/include/python3.8/Python.h
-export PYTHON_VER=python3.8
-export SYS_PYTHON=$PYTHON38_PATH/bin/python3.8
-export SQLITE3_PATH=${SQLITE3_PATH:="${TOOLS_HOME}/sqlite/sqlite3"}
+export PATH="$PYTHON38_PATH/bin:$PATH"
+export SYS_PYTHON="$PYTHON38_PATH/bin/python3.8"
+export SYS_PIP="$PYTHON38_PATH/bin/pip3.8"
+export SQLITE3_PATH=${SQLITE3_PATH:="${TOOLS_HOME}/sqlite/bin/sqlite3"}
 export ORACLE_INSTANTCLIENT19_PATH="/opt/toolchain/instantclient_19_15"
 export LD_LIBRARY_PATH=/usr/local/lib:$ORACLE_INSTANTCLIENT19_PATH:$LD_LIBRARY_PATH
 export LD_RUN_PATH=/usr/local/lib:$ORACLE_INSTANTCLIENT19_PATH:$LD_RUN_PATH
-export PATH=$HOME/.local/bin:$PYTHON38_PATH/bin:${TOOLS_HOME}/sqlite:/usr/bin:$PATH
-
-if [[ $DOCKEROS == "redhat9" || $DOCKEROS == "redhat9_ppc" || $DOCKEROS == "redhat8" || $DOCKEROS == "redhat8-arm64" ]]; then
-  export PYTHON_H=$PYTHON39_PATH/include/python3.9/Python.h
-  export PYTHON_VER=python3.9
-  export SYS_PYTHON=$PYTHON39_PATH/bin/python3.9
-  export PATH=$PYTHON39_PATH/bin:$PATH
-elif [[ $DOCKEROS == "ubuntu22" || $DOCKEROS == "sles15" ]]; then
-  export PYTHON_H=$PYTHON310_PATH/include/python3.10/Python.h
-  export PYTHON_VER=python3.10
-  export SYS_PYTHON=$PYTHON310_PATH/bin/python3.10
-  export PATH=$PYTHON310_PATH/bin:$PATH
-fi
-
-HUE_SRC=$(realpath $WORK_DIR/../..)
-export ROOT=$HUE_SRC
-cd $HUE_SRC
-
-big_console_header "Hue Build Start" "$@"
-make apps docs
-bash -x ./tools/relocatable.sh
-make prod
-big_console_header "Hue Build End" "$@"
+
+PYTHON_VERSIONS=("python3.11" "python3.9" "python3.8")
+for PYTHON_VER in "${PYTHON_VERSIONS[@]}"; do
+  if [[ $PYTHON_VER == "python3.8" && ( $DOCKEROS == "redhat7_ppc" || $DOCKEROS == "redhat8" || $DOCKEROS == "redhat8_ppc" || $DOCKEROS == "sles12" || $DOCKEROS == "centos7" || $DOCKEROS == "ubuntu18" || $DOCKEROS == "ubuntu20" || $DOCKEROS == "ubuntu22" ) ]]; then
+    check_python38_path
+    export PATH="$PYTHON38_PATH/bin:$PATH"
+    export SYS_PYTHON="$PYTHON38_PATH/bin/python3.8"
+    export SYS_PIP="$PYTHON38_PATH/bin/pip3.8"
+    export VIRTUAL_ENV_VERSION="20.24.4"
+  elif [[ $PYTHON_VER == "python3.9" && ( $DOCKEROS == "redhat9" || $DOCKEROS == "redhat8" || $DOCKEROS == "redhat9_ppc" || $DOCKEROS == "redhat8-arm64" ) ]]; then
+    check_python39_path
+    export PATH="$PYTHON39_PATH/bin:$PATH"
+    export SYS_PYTHON="$PYTHON39_PATH/bin/python3.9"
+    export SYS_PIP="$PYTHON39_PATH/bin/pip3.9"
+    export VIRTUAL_ENV_VERSION="20.19.0"
+  elif [[ $PYTHON_VER == "python3.11" && ( $DOCKEROS == "redhat9" || $DOCKEROS == "redhat8" || $DOCKEROS == "sles15" || $DOCKEROS == "redhat8-arm64" ) ]]; then
+    check_python311_path
+    export PATH="$PYTHON311_PATH/bin:$PATH"
+    export SYS_PYTHON="$PYTHON311_PATH/bin/python3.11"
+    export SYS_PIP="$PYTHON311_PATH/bin/pip3.11"
+    export VIRTUAL_ENV_VERSION="20.24.4"
+  else
+    continue
+  fi
+
+  big_console_header "Hue PreRequisite Start for" $PYTHON_VER "$@"
+  install_prerequisite $DOCKEROS
+  big_console_header "Hue PreRequisite End for" $PYTHON_VER "$@"
+
+  HUE_SRC=$(realpath "$WORK_DIR/../..")
+  cd "$HUE_SRC" || exit 1
+
+  BLD_DIR="${BLD_DIR:-build}"             # Default build directory if not set
+  if [[ $PYTHON_VER == "python3.11" ]]; then
+    BLD_DIR_ENV="${BLD_DIR}/env"
+  else
+    BLD_DIR_ENV="${BLD_DIR}/venvs/${PYTHON_VER}"
+  fi
+  echo "BLD_DIR_ENV=${BLD_DIR_ENV}"
+
+  big_console_header "Hue Build Start for" $PYTHON_VER "$@"
+  PYTHON_VER=$PYTHON_VER make apps docs relocatable-env huecheck
+  big_console_header "Hue Build End for" $PYTHON_VER "$@"
+done
+big_console_header "Hue PROD Build Start for" "$@"
+make release
+big_console_header "Hue PROD Build End for" "$@"

+ 73 - 57
tools/cloudera/build_hue_common.sh

@@ -20,7 +20,7 @@ function check_python38_path() {
     echo "Python38 bin does not exists at " $python38_bin
     exit 1
   fi
-  export pip38_bin="$PYTHON38_PATH/bin/pip3.8"
+  export pip_bin="$PYTHON38_PATH/bin/pip3.8"
 }
 
 function check_python39_path() {
@@ -29,7 +29,7 @@ function check_python39_path() {
     echo "Python39 bin does not exists at " $python39_bin
     exit 1
   fi
-  export pip39_bin="$PYTHON39_PATH/bin/pip3.9"
+  export pip_bin="$PYTHON39_PATH/bin/pip3.9"
 }
 
 function check_python310_path() {
@@ -38,7 +38,16 @@ function check_python310_path() {
     echo "Python310 bin does not exists at " $python310_bin
     exit 1
   fi
-  export pip310_bin="$PYTHON310_PATH/bin/pip3.10"
+  export pip_bin="$PYTHON310_PATH/bin/pip3.10"
+}
+
+function check_python311_path() {
+  export python311_bin="$PYTHON311_PATH/bin/python3.11"
+  if [ ! -e "$python311_bin" ]; then
+    echo "Python311 bin does not exists at " $python311_bin
+    exit 1
+  fi
+  export pip_bin="$PYTHON311_PATH/bin/pip3.11"
 }
 
 function check_sqlite3() {
@@ -76,7 +85,7 @@ function redhat7_ppc_install() {
     # sqlite3 install
     sudo TOOLS_HOME=${TOOLS_HOME} -- sh -c 'mkdir -p ${TOOLS_HOME} && \
       cd ${TOOLS_HOME} && \
-      curl -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
+      curl -o sqlite-autoconf-3350500.tar.gz https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
       tar zxvf sqlite-autoconf-3350500.tar.gz && \
       cd sqlite-autoconf-3350500 && \
       ./configure --prefix=${TOOLS_HOME}/sqlite && make && make install'
@@ -90,8 +99,8 @@ function redhat7_ppc_install() {
       ./configure --enable-shared --prefix=/opt/cloudera/cm-agent && \
       make install'
     # Pip modules install
-    sudo pip38_bin=${pip38_bin} -- sh -c '${pip38_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}''
-    sudo pip38_bin=${pip38_bin} -- sh -c 'ln -fs ${pip38_bin} $(dirname ${pip38_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}''
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
   fi
 }
 
@@ -133,13 +142,13 @@ function redhat8_ppc_install() {
     echo 'export PATH=/usr/local/lib/nodejs/node-v20.11.0-linux-ppc64le/bin:$PATH' >> ~/.bashrc
     source ~/.bashrc
     # Pip modules install
-    sudo pip38_bin=${pip38_bin} -- sh -c '${pip38_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
-    sudo pip38_bin=${pip38_bin} -- sh -c '${pip38_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
-    sudo pip38_bin=${pip38_bin} -- sh -c 'ln -fs ${pip38_bin} $(dirname ${pip38_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
     # sqlite3 install
     sudo TOOLS_HOME=${TOOLS_HOME} -- sh -c 'mkdir -p ${TOOLS_HOME} && \
       cd ${TOOLS_HOME} && \
-      curl -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
+      curl -o sqlite-autoconf-3350500.tar.gz https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
       tar zxvf sqlite-autoconf-3350500.tar.gz && \
       cd sqlite-autoconf-3350500 && \
       ./configure --prefix=${TOOLS_HOME}/sqlite && make && make install'
@@ -175,13 +184,13 @@ function redhat9_ppc_install() {
     echo 'export PATH=/usr/local/lib/nodejs/node-v20.11.0-linux-ppc64le/bin:$PATH' >> ~/.bashrc
     source ~/.bashrc
     # Pip modules install
-    sudo pip39_bin=${pip39_bin} -- sh -c '${pip39_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
-    sudo pip39_bin=${pip39_bin} -- sh -c '${pip39_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
-    sudo pip39_bin=${pip39_bin} -- sh -c 'ln -fs ${pip39_bin} $(dirname ${pip39_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
     # sqlite3 install
     sudo TOOLS_HOME=${TOOLS_HOME} -- sh -c 'mkdir -p ${TOOLS_HOME} && \
       cd ${TOOLS_HOME} && \
-      curl -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
+      curl -o sqlite-autoconf-3350500.tar.gz https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
       tar zxvf sqlite-autoconf-3350500.tar.gz && \
       cd sqlite-autoconf-3350500 && \
       ./configure --prefix=${TOOLS_HOME}/sqlite && make && make install'
@@ -192,6 +201,7 @@ function redhat9_ppc_install() {
 function sles12_install() {
   if [[ $FORCEINSTALL -eq 1 ]]; then
     # pre-req install
+    sudo -- sh -c 'zypper refresh'
     sudo -- sh -c 'zypper install -y cyrus-sasl-gssapi \
       cyrus-sasl-plain \
       java-11-openjdk \
@@ -218,11 +228,11 @@ function sles12_install() {
     # Node-v20-LTS is not supported by old OS'es - Redhat7_ppc, Centos7, Ubuntu18, Sles12. So upgrading to node-v16
     sudo -- sh -c 'zypper install -y npm14 nodejs16'
     # Pip modules install
-    sudo pip38_bin=${pip38_bin} -- sh -c '${pip38_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
-    sudo pip38_bin=${pip38_bin} -- sh -c '${pip38_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
-    sudo pip38_bin=${pip38_bin} -- sh -c 'ln -fs ${pip38_bin} $(dirname ${pip38_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
     # sqlite3 install
-    sudo -- sh -c 'curl --insecure -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
+    sudo -- sh -c 'curl --insecure -o sqlite-autoconf-3350500.tar.gz https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
         tar zxvf sqlite-autoconf-3350500.tar.gz && \
         cd sqlite-autoconf-3350500 && \
         ./configure --prefix=/usr/local/ && make && make install'
@@ -233,6 +243,7 @@ function sles12_install() {
 function sles15_install() {
   if [[ $FORCEINSTALL -eq 1 ]]; then
     # pre-req install
+    sudo -- sh -c 'zypper refresh'
     sudo -- sh -c 'zypper install -y cyrus-sasl-gssapi \
       cyrus-sasl-plain \
       java-11-openjdk \
@@ -256,13 +267,13 @@ function sles15_install() {
     # NODEJS 18 install
     sudo -- sh -c 'zypper install -y nodejs18 npm20'
     # Pip modules install
-    sudo pip310_bin=${pip310_bin} -- sh -c '${pip310_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
-    sudo pip310_bin=${pip310_bin} -- sh -c '${pip310_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
-    sudo pip310_bin=${pip310_bin} -- sh -c 'ln -fs ${pip310_bin} $(dirname ${pip310_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
     # sqlite3 install
-    sudo -- sh -c 'curl --insecure -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
-        tar zxvf sqlite-autoconf-3350500.tar.gz && \
-        cd sqlite-autoconf-3350500 && \
+    sudo -- sh -c 'curl --insecure -o sqlite-autoconf-3450000.tar.gz https://sqlite.org/2024/sqlite-autoconf-3450000.tar.gz && \
+        tar zxvf sqlite-autoconf-3450000.tar.gz && \
+        cd sqlite-autoconf-3450000 && \
         ./configure --prefix=/usr/local/ && make && make install'
     export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
   fi
@@ -293,7 +304,7 @@ function centos7_install() {
     fi
     echo "PG_CONFIG is set to $PG_CONFIG"
     # MySQLdb install
-    sudo -- sh -c 'curl -sSLO https://dev.mysql.com/get/mysql80-community-release-el7-11.noarch.rpm && \
+    sudo -- sh -c 'curl -sSLO https://cloudera-build-us-west-1.vpc.cloudera.com/s3/ARTIFACTS/mysql80-community-release-el7-11.noarch.rpm && \
         rpm -ivh mysql80-community-release-el7-11.noarch.rpm && \
         yum install -y mysql-community-libs mysql-community-client-plugins mysql-community-common'
     # NODEJS 16 install
@@ -302,7 +313,7 @@ function centos7_install() {
     sudo -- sh -c 'curl -fsSL https://rpm.nodesource.com/setup_16.x | sudo bash - && \
         yum install -y nodejs npm'
     # sqlite3 install
-    sudo -- sh -c 'curl -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
+    sudo -- sh -c 'curl -o sqlite-autoconf-3350500.tar.gz https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
         tar zxvf sqlite-autoconf-3350500.tar.gz && \
         cd sqlite-autoconf-3350500 && \
         ./configure --prefix=/usr/local/ && make && make install'
@@ -315,9 +326,9 @@ function centos7_install() {
       ./configure --enable-shared --prefix=/opt/cloudera/cm-agent && \
       make install'
     # Pip modules install
-    sudo pip38_bin=${pip38_bin} -- sh -c '${pip38_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
-    sudo pip38_bin=${pip38_bin} -- sh -c '${pip38_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
-    sudo pip38_bin=${pip38_bin} -- sh -c 'ln -fs ${pip38_bin} $(dirname ${pip38_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
   fi
 }
 
@@ -350,11 +361,11 @@ function redhat8_install() {
     sudo -- sh -c 'curl -fsSL https://rpm.nodesource.com/setup_lts.x | sudo bash - && \
       yum install -y nodejs'
     # Pip modules install
-    sudo pip39_bin=${pip39_bin} -- sh -c '${pip39_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
-    sudo pip39_bin=${pip39_bin} -- sh -c '${pip39_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
-    sudo pip39_bin=${pip39_bin} -- sh -c 'ln -fs ${pip39_bin} $(dirname ${pip39_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
     # sqlite3 install
-    sudo -- sh -c 'curl -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
+    sudo -- sh -c 'curl -o sqlite-autoconf-3350500.tar.gz https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
       tar zxvf sqlite-autoconf-3350500.tar.gz && \
       cd sqlite-autoconf-3350500 && \
       ./configure --prefix=/usr/local/ && make && make install'
@@ -382,11 +393,11 @@ function redhat8_arm64_install() {
     sudo -- sh -c 'curl -fsSL https://rpm.nodesource.com/setup_lts.x | sudo bash - && \
       yum install -y nodejs'
     # Pip modules install
-    sudo pip39_bin=${pip39_bin} -- sh -c '${pip39_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
-    sudo pip39_bin=${pip39_bin} -- sh -c '${pip39_bin} install psycopg2-binary==2.9.6'
-    sudo pip39_bin=${pip39_bin} -- sh -c 'ln -fs ${pip39_bin} $(dirname ${pip39_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install psycopg2-binary==2.9.6'
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
     # sqlite3 install
-    sudo -- sh -c 'curl -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
+    sudo -- sh -c 'curl -o sqlite-autoconf-3350500.tar.gz https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
       tar zxvf sqlite-autoconf-3350500.tar.gz && \
       cd sqlite-autoconf-3350500 && \
       ./configure --prefix=/usr/local/ && make && make install'
@@ -397,6 +408,7 @@ function redhat8_arm64_install() {
 function ubuntu18_install() {
     if [[ $FORCEINSTALL -eq 1 ]]; then
     # pre-req install
+    sudo -- sh -c 'apt-get update'
     sudo -- sh -c 'DEBIAN_FRONTEND=noninteractive apt -qq -y install  \
         krb5-user \
         krb5-kdc \
@@ -438,11 +450,11 @@ function ubuntu18_install() {
     sudo -- sh -c 'curl -sL https://deb.nodesource.com/setup_16.x | sudo bash - && \
       apt -y install nodejs'
     # Pip modules install
-    sudo pip38_bin=${pip38_bin} -- sh -c '${pip38_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
-    sudo pip38_bin=${pip38_bin} -- sh -c '${pip38_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
-    sudo pip38_bin=${pip38_bin} -- sh -c 'ln -fs ${pip38_bin} $(dirname ${pip38_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
     # sqlite3 install
-    sudo -- sh -c 'curl -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
+    sudo -- sh -c 'curl -o sqlite-autoconf-3350500.tar.gz https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
         tar zxvf sqlite-autoconf-3350500.tar.gz && \
         cd sqlite-autoconf-3350500 && \
         ./configure --prefix=/usr/local/ && make && make install'
@@ -495,11 +507,11 @@ function ubuntu20_install() {
       curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - && \
       apt-get install -y nodejs'
     # Pip modules install
-    sudo pip38_bin=${pip38_bin} -- sh -c '${pip38_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
-    sudo pip38_bin=${pip38_bin} -- sh -c '${pip38_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
-    sudo pip38_bin=${pip38_bin} -- sh -c 'ln -fs ${pip38_bin} $(dirname ${pip38_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
     # sqlite3 install
-    sudo -- sh -c 'curl -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
+    sudo -- sh -c 'curl -o sqlite-autoconf-3350500.tar.gz https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
         tar zxvf sqlite-autoconf-3350500.tar.gz && \
         cd sqlite-autoconf-3350500 && \
         ./configure --prefix=/usr/local/ && make && make install'
@@ -509,6 +521,10 @@ function ubuntu20_install() {
 
 function ubuntu22_install() {
     if [[ $FORCEINSTALL -eq 1 ]]; then
+    sudo -- sh -c 'apt update'
+    # Add deadsnakes PPA for Python 3.8
+    sudo -- sh -c 'add-apt-repository ppa:deadsnakes/ppa -y'
+    sudo -- sh -c 'apt update'
     # pre-req install
     sudo -- sh -c 'DEBIAN_FRONTEND=noninteractive apt -qq -y install  \
         krb5-user \
@@ -517,9 +533,9 @@ function ubuntu22_install() {
         libkrb5-dev'
     sudo -- sh -c 'apt -y install \
         ldap-utils \
-        libpython3.10-dev \
-        libpython3.10-minimal \
-        libpython3.10-stdlib \
+        libpython3.8-dev \
+        libpython3.8-minimal \
+        libpython3.8-stdlib \
         libxmlsec1 \
         libxmlsec1-openssl \
         libpq-dev \
@@ -531,7 +547,7 @@ function ubuntu22_install() {
         python3-psycopg2 \
         python3-setuptools \
         python3-wheel \
-        python3.10-venv \
+        python3.8-venv \
         openssl \
         sudo \
         tar \
@@ -550,11 +566,11 @@ function ubuntu22_install() {
       curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - && \
       apt-get install -y nodejs'
     # Pip modules install
-    sudo pip310_bin=${pip310_bin} -- sh -c '${pip310_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
-    sudo pip310_bin=${pip310_bin} -- sh -c '${pip310_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
-    sudo pip310_bin=${pip310_bin} -- sh -c 'ln -fs ${pip310_bin} $(dirname ${pip310_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
     # sqlite3 install
-    sudo -- sh -c 'curl -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
+    sudo -- sh -c 'curl -o sqlite-autoconf-3350500.tar.gz https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
         tar zxvf sqlite-autoconf-3350500.tar.gz && \
         cd sqlite-autoconf-3350500 && \
         ./configure --prefix=/usr/local/ && make && make install'
@@ -590,11 +606,11 @@ function redhat9_install() {
     sudo -- sh -c 'curl -fsSL https://rpm.nodesource.com/setup_lts.x | sudo bash - && \
       yum install -y nodejs'
     # Pip modules install
-    sudo pip39_bin=${pip39_bin} -- sh -c '${pip39_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
-    sudo pip39_bin=${pip39_bin} -- sh -c '${pip39_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
-    sudo pip39_bin=${pip39_bin} -- sh -c 'ln -fs ${pip39_bin} $(dirname ${pip39_bin})/pip'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install virtualenv=='${VIRTUAL_ENV_VERSION}' virtualenv-make-relocatable=='${VIRTUAL_ENV_RELOCATABLE_VERSION}' mysqlclient==2.1.1'
+    sudo pip_bin=${pip_bin} -- sh -c '${pip_bin} install psycopg2==2.9.6 --global-option=build_ext --global-option="--pg-config=$PG_CONFIG"'
+    sudo pip_bin=${pip_bin} -- sh -c 'ln -fs ${pip_bin} $(dirname ${pip_bin})/pip'
     # sqlite3 install
-    sudo -- sh -c 'curl -Lo sqlite-autoconf-3350500.tar.gz https://www.sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
+    sudo -- sh -c 'curl -o sqlite-autoconf-3350500.tar.gz https://sqlite.org/2021/sqlite-autoconf-3350500.tar.gz && \
       tar zxvf sqlite-autoconf-3350500.tar.gz && \
       cd sqlite-autoconf-3350500 && \
       ./configure --prefix=/usr/local/ && make && make install'

+ 74 - 0
tools/relocatable.py

@@ -0,0 +1,74 @@
+"""
+Rewrites absolute paths in `.pth` files inside a Python virtualenv to be relative.
+
+Used to ensure that Hue's virtual environments are relocatable across systems.
+Complements `virtualenv-make-relocatable` for robust deployment support.
+"""
+
+import os
+import site
+import shutil
+from pathlib import Path
+
+
+def relocate_pth_with_relative_paths(site_packages: str, make_backup: bool = True) -> None:
+    """
+    Converts absolute paths in .pth files to relative paths, relative to the site-packages directory.
+    Only modifies paths that:
+      - Exist on the filesystem.
+      - Are not already within the site-packages directory.
+    Ignores empty lines and lines starting with 'import'.
+
+    Parameters:
+        site_packages (str): Path to the site-packages directory.
+        make_backup (bool): Whether to save a backup copy of each modified .pth file.
+    """
+    site_packages = Path(site_packages).resolve()
+    pth_files = site_packages.glob("*.pth")
+
+    for pth_file in pth_files:
+        updated_lines = []
+        modified = False
+
+        with pth_file.open("r") as f:
+            for line in f:
+                stripped = line.strip()
+
+                # Preserve empty lines and import statements
+                if not stripped or stripped.startswith("import"):
+                    updated_lines.append(line.rstrip())
+                    continue
+
+                path = Path(stripped)
+                if path.is_absolute() and path.exists():
+                    # Only convert if path is outside site-packages
+                    if site_packages not in path.parents:
+                        try:
+                            # Compute relative path to site-packages
+                            relative_path = os.path.relpath(path, site_packages)
+                            updated_lines.append(relative_path)
+                            modified = True
+                        except Exception as e:
+                            print(f"️Failed to compute relative path for: {path}\n   Error: {e}")
+                            updated_lines.append(stripped)  # fallback: keep original
+                    else:
+                        updated_lines.append(stripped)
+                else:
+                    updated_lines.append(stripped)
+
+        if modified:
+            if make_backup:
+                backup_path = pth_file.with_suffix(pth_file.suffix + ".bak")
+                shutil.copy(pth_file, backup_path)
+
+            with pth_file.open("w") as f:
+                f.write("\n".join(updated_lines) + "\n")
+
+            print(f"Rewritten with relative paths: {pth_file.name}")
+        else:
+            print(f"No changes needed: {pth_file.name}")
+
+
+if __name__ == "__main__":
+    site_packages_dir = site.getsitepackages()[0]
+    relocate_pth_with_relative_paths(site_packages_dir)

+ 17 - 7
tools/relocatable.sh

@@ -22,13 +22,20 @@ usage() {
   Make a Hue installation relocatable. Run this in the installation
   directory created by 'make install', before you relocate it.
   Usage:
-      $0
+    ./tools/relocatable.sh [python_version]
+  Example:
+    ./tools/relocatable.sh python3.9
   "
   exit 1
 }
 
-if [ $# != 0 ] ; then
+# Check for arguments
+PYTHON_VER=""
+if [ $# -gt 1 ]; then
   usage
+elif [ $# -eq 1 ]; then
+  PYTHON_VER=$1
+  echo "Using Python version: $PYTHON_VER"
 fi
 
 find_os() {
@@ -66,7 +73,9 @@ elif [[ $CURR_DIR == */tools ]]; then
   HUE_ROOT=$(dirname $CURR_DIR)
 fi
 
-ENV_PYTHON="$HUE_ROOT/build/env/bin/python"
+BLD_DIR_BIN="$BLD_DIR_ENV/bin"
+ENV_PYTHON="$BLD_DIR_BIN/python"
+
 VIRTUAL_BOOTSTRAP="$CURR_DIR/virtual-bootstrap/virtual-bootstrap.py"
 
 if [[ ! -e $ENV_PYTHON ]]; then
@@ -84,18 +93,19 @@ fi
 export PATH=$(dirname $ENV_PYTHON):$PATH
 
 PYVER=$($ENV_PYTHON -V 2>&1 | awk '{print $2}' | cut -d '.' -f 1,2)
+BIN_DIR=$(dirname $SYS_PYTHON)
 # Step 1. Fix virtualenv
-if [[ "$PYVER" == "3."[8-9]* || "$PYVER" == "3.10" ]]; then
+if [[ "$PYVER" == "3."[8-9]* || "$PYVER" == "3.11" ]]; then
   echo "Python version is 3.8 or greater"
   pushd .
   cd $HUE_ROOT
-  virtualenv-make-relocatable "build/env"
-  $ENV_PYTHON $VIRTUAL_BOOTSTRAP --relocatable_pth "build/env"
+  $SYS_PYTHON $BIN_DIR/virtualenv-make-relocatable "$BLD_DIR_ENV"
+  # $SYS_PYTHON $VIRTUAL_BOOTSTRAP --relocatable_pth "$BLD_ENV_REL"
   popd
 fi
 
 # Step 1b. Fix any broken lib64 directory
-LIB64="$HUE_ROOT/build/env/lib64"
+LIB64="$HUE_ROOT/$BLD_ENV_REL/lib64"
 if [ -L "$LIB64" -a ! -e "$LIB64" ] ; then
   rm "$LIB64"
   ln -s lib "$LIB64"