Преглед на файлове

HUE-8737 [core] make Hue install third party libs via requirement.txt

Ying Chen преди 6 години
родител
ревизия
9cf00ec3ad
променени са 9 файла, в които са добавени 173 реда и са изтрити 12 реда
  1. 1 0
      Makefile
  2. 10 0
      Makefile.sdk
  3. 4 0
      Makefile.vars
  4. 1 0
      desktop/Makefile
  5. 5 1
      desktop/core/Makefile
  6. 109 0
      desktop/core/requirements_py3.txt
  7. 30 11
      package-lock.json
  8. 11 0
      tools/app_reg/common.py
  9. 2 0
      tools/app_reg/registry.py

+ 1 - 0
Makefile

@@ -145,6 +145,7 @@ $(BLD_DIR_ENV)/stamp:
 desktop: parent-pom
 # END DEV ONLY >>>>
 desktop: virtual-env
+	@echo ">>>-----> desktop: "
 	@$(MAKE) -C desktop
 
 

+ 10 - 0
Makefile.sdk

@@ -109,6 +109,16 @@ $(EXT_PY_EGG_TARGETS): CUR_EXT_PY=$(notdir $(@D))
 $(EXT_PY_CLEAN_TARGETS): CUR_EXT_PY=$(notdir $(@D))
 $(EXT_PY_ENV_INSTALL_TARGETS): CUR_EXT_PY=$(notdir $(@D))
 
+#
+# ext-env-pip-install
+#   	Pip install any 3rd party package that are based on requirements_py3.txt
+#
+.PHONY: ext-env-pip-install
+ext-env-pip-install:
+	@echo '--- Installing requirement.txt into virtual-env via $(ENV_PIP) command to file $(REQUIREMENT_FILE).'
+	@$(ENV_PIP) install -r $(REQUIREMENT_FILE)
+	@echo '--- Finished requirement.txt into virtual-env'
+
 #
 # ext-eggs
 #   	Builds a binary egg for any 3rd party package that are based on the

+ 4 - 0
Makefile.vars

@@ -44,12 +44,14 @@ PYTHON_EXE = $(shell echo $(PYTHON_VER) | sed "s/\.//")
 
 ifeq ($(PYTHON_VER),python2.7)
   $(info "PYTHNON_VER is $(PYTHON_VER)")
+  EXT_ENV_INSTALL = ext-env-install
 else ifeq ($(shell echo $(PYTHON_VER) | head -c 8),python3.)
   PYTHON_VER := $(shell echo $(PYTHON_VER) | head -c 9)
   MINOR_VER = $(shell echo -n $(PYTHON_VER) | tail -c 1)
   ifeq ($(shell test $(MINOR_VER) -lt 5; echo $$?),0)
     $(error "$(VER_ERR_MSG)")
   endif
+  EXT_ENV_INSTALL = ext-env-pip-install
 else
   $(error "$(VER_ERR_MSG)")
 endif
@@ -97,6 +99,7 @@ ENV_PYTHON_VERSION = $(shell $(ENV_PYTHON) -c 'import sys; print ("python%d.%d"%
 # doesn't arise.
 ##############################
 ENV_EASY_INSTALL := $(ENV_PYTHON) $(BLD_DIR_BIN)/easy_install
+ENV_PIP := $(ENV_PYTHON) $(BLD_DIR_BIN)/pip
 
 ##############################
 # This version is substituted through to the tarballs and packages.
@@ -114,3 +117,4 @@ DB_PROXY_JAR := $(ROOT)/desktop/libs/librdbms/java-lib/dbproxy-1.0.jar
 # Internationalization
 ################################################
 PYBABEL := $(ROOT)/build/env/bin/pybabel
+REQUIREMENT_FILE := $(ROOT)/desktop/core/requirements_py3.txt

+ 1 - 0
desktop/Makefile

@@ -102,6 +102,7 @@ $(DESKTOP_DB): $(BLD_DIR_BIN)/hue
 
 # Targets that simply recurse into all of the applications
 ENV_INSTALL_TARGETS := $(APPS:%=.recursive-env-install/%)
+$(info "----------------> ENV_INSTALL_TARGETS: $(ENV_INSTALL_TARGETS)")
 .recursive-env-install/%: %
 	$(MAKE) -C $< env-install
 

+ 5 - 1
desktop/core/Makefile

@@ -20,6 +20,7 @@ ifeq ($(ROOT),)
   $(error "Error: Expect the environment variable $$ROOT to point to the Desktop installation")
 endif
 
+
 # The default RSYNC_OPT copies unsafe links. We don't want that because
 # the symlink targets are always installed (separately). See HUE-565.
 RSYNC_OPT := -a
@@ -27,12 +28,15 @@ include $(ROOT)/Makefile.sdk
 
 default::
 	@echo '  env-install    : Install into virtual-env'
+	@echo 'desktop/core/Makefile default EXT_ENV_INSTALL: $(EXT_ENV_INSTALL)'
 
 #
 # env-install
 # 	Install app into the virtual environment.
 #
 .PHONY: env-install
-env-install: compile ext-env-install
+env-install: compile $(EXT_ENV_INSTALL)
 	@echo '--- Installing $(APP_NAME) into virtual-env'
+	@echo '--- env-install: EXT_ENV_INSTALL: $(EXT_ENV_INSTALL)'
 	@$(ENV_PYTHON) setup.py develop -N -q
+

+ 109 - 0
desktop/core/requirements_py3.txt

@@ -0,0 +1,109 @@
+SQLAlchemy==1.3.8
+ipaddress==1.0.19
+python-snappy==0.5.4
+urllib3==1.22
+django-celery-beat==1.4.0
+django-extensions==1.8.0
+monotonic==1.5
+python-ldap==3.1.0
+djangomako==1.0.1
+lockfile==0.12.2
+chardet==3.0.4
+defusedxml==0.5.0
+gunicorn==19.9.0
+vine==1.2.0
+MarkupSafe==0.9.3
+backports.csv==1.0.5
+django-debug-toolbar==1.9.1
+jaeger-client==4.0.0
+thriftpy==0.3.9
+pyasn1==0.4.5
+et_xmlfile==1.0.1
+wheel==0.31.0
+boto==2.46.1
+rsa==3.4.2
+Markdown==3.1
+odfpy==1.4.0
+Mako==1.0.7
+celery==4.2.1
+pyOpenSSL==17.5.0
+PySocks==1.7.0
+tablib==0.13.0
+ndg_httpsclient==0.4.0
+Django==1.11.20
+python-crontab==2.3.6
+enum-compat==0.0.2
+colorama==0.3.2
+pyformance==0.3.2
+kerberos==1.3.0
+requests-kerberos==0.12.0
+pytz==2015.2
+python-dateutil==2.4.2
+docutils==0.14
+Paste==2.0.1
+josepy==1.1.0
+importlib2==3.5.0.2
+ordereddict==1.1
+httplib2==0.8
+Pygments==1.3.1
+py4j==0.9
+pysaml2==4.4.0
+Babel==2.5.1
+configobj==5.0.6
+mysqlclient==1.4.2.post1
+avro-python3==1.8.2
+dnspython==1.15.0
+amqp==2.4.1
+django-auth-ldap==1.3.0
+PyYAML==3.12
+kombu==4.3.0
+jdcal==1.0.1
+pycparser==2.18
+django-webpack-loader==0.5.0
+django_celery_results==1.0.4
+django_opentracing==1.1.0
+django_prometheus==1.0.15
+pyasn1-modules==0.2.4
+django-openid-auth==0.14
+thrift==0.11.0
+billiard==3.5.0.5
+simple-salesforce==0.74.2
+sasl==0.2.1
+django-axes==2.2.0
+sqlparse==0.2.0
+eventlet==0.24.1
+openpyxl==2.6.2
+pytidylib==0.3.2
+django-nose==1.4.5
+simplejson==3.15.0
+cx_Oracle==7.1.2
+idna==2.6
+cffi==1.11.5
+future==0.17.1
+django-debug-panel==0.8.3
+xlwt==1.3.0
+parquet==1.1
+mozilla-django-oidc==1.0.0
+xlrd==1.2.0
+six==1.12.0
+lxml==3.3.6
+django-timezone-field==3.0
+pycryptodomex==3.4.7
+zope.interface==4.5.0
+python-oauth2==1.1.0
+python-daemon==2.2.3
+nose==1.3.7
+python-pam==1.8.4
+enum34==1.1.6
+requests==2.18.4
+repoze.who==2.3
+djangosaml2==0.16.11
+asn1crypto==0.24.0
+greenlet==0.4.15
+BabelDjango==0.2.2
+certifi==2018.1.18
+pycrypto==2.6.1
+cryptography==2.1.4
+ply==3.9
+kazoo==2.0
+gcs-oauth2-boto-plugin==2.5

+ 30 - 11
package-lock.json

@@ -5097,7 +5097,8 @@
         "ansi-regex": {
           "version": "2.1.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "aproba": {
           "version": "1.2.0",
@@ -5118,12 +5119,14 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -5138,17 +5141,20 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -5265,7 +5271,8 @@
         "inherits": {
           "version": "2.0.3",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -5277,6 +5284,7 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -5291,6 +5299,7 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -5298,12 +5307,14 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "minipass": {
           "version": "2.3.5",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.2",
             "yallist": "^3.0.0"
@@ -5322,6 +5333,7 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -5402,7 +5414,8 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -5414,6 +5427,7 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -5499,7 +5513,8 @@
         "safe-buffer": {
           "version": "5.1.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "safer-buffer": {
           "version": "2.1.2",
@@ -5535,6 +5550,7 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -5554,6 +5570,7 @@
           "version": "3.0.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "ansi-regex": "^2.0.0"
           }
@@ -5597,12 +5614,14 @@
         "wrappy": {
           "version": "1.0.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "yallist": {
           "version": "3.0.3",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         }
       }
     },

+ 11 - 0
tools/app_reg/common.py

@@ -62,3 +62,14 @@ def _get_python_lib_dir():
 
 def _get_python_site_packages_dir():
   return os.path.join(_get_python_lib_dir(), 'site-packages')
+
+def cmp(x, y):
+  """
+  Replacement for built-in function cmp that was removed in Python 3
+
+  Compare the two objects x and y and return an integer according to
+  the outcome. The return value is negative if x < y, zero if x == y
+  and strictly positive if x > y.
+  """
+
+  return (x > y) - (x < y)

+ 2 - 0
tools/app_reg/registry.py

@@ -19,6 +19,7 @@
 Registry for the applications
 """
 
+from builtins import object
 import glob
 import logging
 import os
@@ -26,6 +27,7 @@ import sys
 import json
 
 import common
+from common import cmp
 
 if sys.version_info[0] > 2:
   from builtins import object