Prechádzať zdrojové kódy

HUE-3287 [core] Django 1.11 upgrade
- HUE-8261 [core] Fix the pypi URL when building

setup Python2.7 installation
Fixing python2.7 shebang issue when virtual_bootstrap --relocatable runs
Moving Python installation from /opt/cloudera to /usr/local due to http://github.mtv.cloudera.com/CDH/cdh/blob/cdh6.x/lib/python/cauldron/src/cauldron/tools/buildparcel.py#L686
Enabling /opt/rh/python27/root/usr/bin/python through scl tool
Adding Python2.7 enabling script
adding python version line
Adding makefile change

Prakash Ranade 7 rokov pred
rodič
commit
3a33f8889b

+ 1 - 0
Makefile

@@ -162,6 +162,7 @@ INSTALL_CORE_FILES = \
 	ext \
 	tools/app_reg \
 	tools/virtual-bootstrap \
+	tools/enable-python27.sh \
 	tools/relocatable.sh \
 	VERS* LICENSE* README*
 

+ 1 - 0
Makefile.tarball

@@ -29,6 +29,7 @@ PROD_INCLUDES := \
 	tools/app_reg \
 	tools/virtual-bootstrap \
 	tools/relocatable.sh \
+	tools/enable-python27.sh \
 	tools/load-balancer \
 	VERSION
 

+ 3 - 3
Makefile.vars

@@ -36,15 +36,15 @@ ifneq ($(wildcard $(ROOT)/Makefile.buildvars),)
   include $(ROOT)/Makefile.buildvars
 endif
 
-PYTHON_H ?= $(shell ls /usr/include/python2.7/Python.h 2>/dev/null || ls /opt/cloudera/python/include/python2.7/Python.h 2>/dev/null)
+PYTHON_H ?= $(shell ls /usr/include/python2.7/Python.h 2>/dev/null || ls /usr/local/python27/include/python2.7/Python.h 2>/dev/null || ls /opt/rh/python27/root/usr/include/python2.7/Python.h 2>/dev/null)
 ifndef SKIP_PYTHONDEV_CHECK
   ifeq ($(PYTHON_H),)
     $(error "Error: must have python development packages for 2.7. Could not find Python.h. Please install python2.7-devel")
   endif
   PYTHON_VER := $(shell echo $(PYTHON_H) | grep -o python2.7)
-  SYS_PYTHON ?= $(shell ls /usr/bin/$(PYTHON_VER) 2>/dev/null || ls /opt/cloudera/python/bin/$(PYTHON_VER) 2>/dev/null)
+  SYS_PYTHON ?= $(shell ls /usr/bin/$(PYTHON_VER) 2>/dev/null || ls /usr/local/python27/bin/$(PYTHON_VER) 2>/dev/null || ls /opt/rh/python27/root/usr/bin/$(PYTHON_VER) 2>/dev/null)
 else
-  SYS_PYTHON ?= $(shell which python)
+  SYS_PYTHON ?= $(shell which python2.7)
 endif
 
 ifeq ($(SYS_PYTHON),)

+ 1 - 19
apps/Makefile

@@ -30,25 +30,7 @@ default: env-install
 
 .PHONY: clean default distclean install
 
-APPS := about \
-  beeswax \
-  metastore \
-  filebrowser \
-  help \
-  jobbrowser \
-  jobsub \
-  oozie \
-  proxy \
-  useradmin \
-  impala \
-  pig \
-  search \
-  hbase \
-  sqoop \
-  zookeeper \
-  rdbms \
-  spark \
-  security
+APPS := $(shell find . -maxdepth 2 -name "src" -type d|cut -d"/" -f2|sort)
 
 ################################################
 # Install all applications into the Desktop environment

+ 4 - 2
desktop/devtools.mk

@@ -31,12 +31,14 @@ DEVTOOLS += \
         isort[4.2.5] \
         six[1.10.0]
 
+PYPI_MIRROR ?= https://pypi.python.org/simple/
+
 # Install/download dev tools for SDK into the virtual environment
 .PHONY: $(DEVTOOLS)
 $(DEVTOOLS):
 	@echo "--- Installing development tool: $@"
-	$(ENV_EASY_INSTALL) -f http://archive.cloudera.com/desktop-sdk-python-packages/ -i https://pypi.infra.cloudera.com/api/pypi/pypi-public/simple/ \
-	   -H archive.cloudera.com,pypi.infra.cloudera.com,pypi.python.org $(SETUPTOOLS_OPTS) $(subst ],,$(subst [,==,$@))
+	$(ENV_EASY_INSTALL) -i $(PYPI_MIRROR) \
+	   -H *.cloudera.com,pypi.python.org,files.pythonhosted.org $(SETUPTOOLS_OPTS) $(subst ],,$(subst [,==,$@))
 
 $(BLD_DIR):
 	@mkdir -p $@

+ 30 - 0
tools/enable-python27.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+python27_exists=0
+python_vers=("/usr/bin" "/usr/local/python27/bin" "/opt/rh/python27/root/usr/bin")
+
+function run_python() {
+  $1 --version >/dev/null 2>&1
+  if [ ! $? -eq 0 ]; then
+    echo 0
+  else
+    echo 1
+  fi
+}
+
+for binpath in ${python_vers[@]}; do
+  if [[ $binpath == "/opt/rh/python27"* ]]; then
+    . /opt/rh/python27/enable
+  fi
+  out=$(run_python $binpath/python2.7)
+  if [ $out -eq 1 ]; then
+    export PATH=$binpath:$PATH
+    python27_exists=1
+    break
+  fi
+done
+
+if [ $python27_exists -eq 0 ]; then
+  echo "ERROR: Unable to find python 2.7 installation"
+  exit 1
+fi

+ 3 - 0
tools/relocatable.sh

@@ -37,6 +37,9 @@ HUE_ROOT="$CURR_DIR/.."
 ENV_PYTHON="$HUE_ROOT/build/env/bin/python"
 VIRTUAL_BOOTSTRAP="$CURR_DIR/virtual-bootstrap/virtual-bootstrap.py"
 
+source $HUE_ROOT/tools/enable-python27.sh
+
+export PATH=$(dirname $ENV_PYTHON):$PATH
 # Step 1. Fix virtualenv
 $ENV_PYTHON $VIRTUAL_BOOTSTRAP --relocatable "$HUE_ROOT/build/env"
 

+ 8 - 1
tools/virtual-bootstrap/virtual-bootstrap.py

@@ -1712,6 +1712,9 @@ def fixup_egg_link(filename):
     if os.path.abspath(link) != link:
         logger.debug('Link in %s already relative' % filename)
         return
+    # filename = /container.ubuntu1604/output/cdh/hue/hue-3.9.0+cdh6.x+0/build/env/local/lib/python2.7/site-packages/liboozie.egg-link
+    # link =     /container.ubuntu1604/output/cdh/hue/hue-3.9.0+cdh6.x+0/desktop/libs/liboozie/src
+    # new_link = ../../../../../../desktop/libs/liboozie/src
     new_link = make_relative_path(filename, link)
     logger.notify('Rewriting link %s in %s as %s' % (link, filename, new_link))
     with open(filename, 'w') as f:
@@ -1742,7 +1745,11 @@ def make_relative_path(source, dest, dest_is_directory=True):
     while dest_parts and source_parts and dest_parts[0] == source_parts[0]:
         dest_parts.pop(0)
         source_parts.pop(0)
-    full_parts = ['..']*len(source_parts) + dest_parts
+    full_parts = ""
+    if '/local/' in source:
+        full_parts = ['..']*(len(source_parts)-1) + dest_parts
+    else:
+        full_parts = ['..']*len(source_parts) + dest_parts
     if not dest_is_directory:
         full_parts.append(dest_filename)
     if not full_parts: