Răsfoiți Sursa

HUE-8737 [py3] Make a lot of non essnential modules optional

Romain 5 ani în urmă
părinte
comite
e4853b6f1e

+ 7 - 3
Makefile.vars

@@ -42,7 +42,7 @@ VER_ERR_MSG = "Variable PYTHON_VER is $(PYTHON_VER) but it only supports python2
 PYTHON_VER ?= python2.7
 PYTHON_VER ?= python2.7
 PYTHON_EXE = $(shell echo $(PYTHON_VER) | sed "s/\.//")
 PYTHON_EXE = $(shell echo $(PYTHON_VER) | sed "s/\.//")
 
 
-$(info "PYTHON_VER is $(PYTHON_VER)")
+$(info "PYTHON_VER is $(PYTHON_VER).")
 
 
 ifeq ($(PYTHON_VER),python2.7)
 ifeq ($(PYTHON_VER),python2.7)
   $(info "Python 2 module install of desktop/ext-py")
   $(info "Python 2 module install of desktop/ext-py")
@@ -71,6 +71,8 @@ endif
 
 
 ifeq ($(SYS_PYTHON),)
 ifeq ($(SYS_PYTHON),)
   $(error "Error: Need python version 2.7 or >= 3.5")
   $(error "Error: Need python version 2.7 or >= 3.5")
+else
+  $(info "SYS_PYTHON is $(SYS_PYTHON).")
 endif
 endif
 
 
 HADOOP_HOME ?= /usr/lib/hadoop
 HADOOP_HOME ?= /usr/lib/hadoop
@@ -94,6 +96,8 @@ STATIC_DIR := $(BLD_DIR)/static
 ENV_PYTHON := $(BLD_DIR_ENV)/bin/$(notdir $(SYS_PYTHON))
 ENV_PYTHON := $(BLD_DIR_ENV)/bin/$(notdir $(SYS_PYTHON))
 ENV_PYTHON_VERSION = $(shell $(ENV_PYTHON) -c 'import sys; print ("python%d.%d"% sys.version_info[:2])')
 ENV_PYTHON_VERSION = $(shell $(ENV_PYTHON) -c 'import sys; print ("python%d.%d"% sys.version_info[:2])')
 
 
+$(info "ENV_PYTHON is $(ENV_PYTHON).")
+
 
 
 ##############################
 ##############################
 # ENV_EASY_INSTALL uses the easy_install script installed in the virtual
 # ENV_EASY_INSTALL uses the easy_install script installed in the virtual
@@ -107,7 +111,7 @@ ENV_PIP := $(ENV_PYTHON) $(BLD_DIR_BIN)/pip
 ##############################
 ##############################
 # This version is substituted through to the tarballs and packages.
 # This version is substituted through to the tarballs and packages.
 ##############################
 ##############################
-DESKTOP_VERSION := $(shell python <(cat $(ROOT)/VERSION; echo print '(VERSION)'))
+DESKTOP_VERSION := $(shell $(SYS_PYTHON) <(cat $(ROOT)/VERSION; echo print '(VERSION)'))
 MAVEN_VERSION = $(DESKTOP_VERSION)-SNAPSHOT
 MAVEN_VERSION = $(DESKTOP_VERSION)-SNAPSHOT
 
 
 
 
@@ -124,5 +128,5 @@ PYBABEL := $(ROOT)/build/env/bin/pybabel
 ##############################
 ##############################
 # Path to files for pip requirements
 # Path to files for pip requirements
 ##############################
 ##############################
-REQUIREMENT_FILE := $(ROOT)/desktop/core/requirements_py3.txt
+REQUIREMENT_FILE := $(ROOT)/desktop/core/requirements.txt
 NAVOPTAPI_WHL := $(ROOT)/desktop/core/wheels/navoptapi-1.0.0-py3-none-any.whl
 NAVOPTAPI_WHL := $(ROOT)/desktop/core/wheels/navoptapi-1.0.0-py3-none-any.whl

+ 18 - 10
apps/useradmin/src/useradmin/ldap_access.py

@@ -18,20 +18,28 @@
 This module provides access to LDAP servers, along with some basic functionality required for Hue and
 This module provides access to LDAP servers, along with some basic functionality required for Hue and
 User Admin to work seamlessly with LDAP.
 User Admin to work seamlessly with LDAP.
 """
 """
-from builtins import str
-from builtins import object
-import ldap
-import ldap.filter
+from builtins import str, object
+
 import logging
 import logging
+
+LOG = logging.getLogger(__name__)
+
+try:
+  import ldap
+  import ldap.filter
+  from ldap import SCOPE_SUBTREE
+except ImportError:
+  LOG.warn('ldap module not found')
+  SCOPE_SUBTREE = None
 import re
 import re
 
 
+from django.utils.encoding import smart_text
+
 import desktop.conf
 import desktop.conf
 from desktop.lib.python_util import CaseInsensitiveDict
 from desktop.lib.python_util import CaseInsensitiveDict
-from useradmin.models import User
-from django.utils.encoding import smart_text
 
 
+from useradmin.models import User
 
 
-LOG = logging.getLogger(__name__)
 CACHED_LDAP_CONN = None
 CACHED_LDAP_CONN = None
 
 
 
 
@@ -301,7 +309,7 @@ class LdapConnection(object):
 
 
     return group_info
     return group_info
 
 
-  def find_users(self, username_pattern, search_attr=None, user_name_attr=None, user_filter=None, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
+  def find_users(self, username_pattern, search_attr=None, user_name_attr=None, user_filter=None, find_by_dn=False, scope=SCOPE_SUBTREE):
     """
     """
     LDAP search helper method finding users. This supports searching for users
     LDAP search helper method finding users. This supports searching for users
     by distinguished name, or the configured username attribute.
     by distinguished name, or the configured username attribute.
@@ -364,7 +372,7 @@ class LdapConnection(object):
 
 
     return None
     return None
 
 
-  def find_groups(self, groupname_pattern, search_attr=None, group_name_attr=None, group_member_attr=None, group_filter=None, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
+  def find_groups(self, groupname_pattern, search_attr=None, group_name_attr=None, group_member_attr=None, group_filter=None, find_by_dn=False, scope=SCOPE_SUBTREE):
     """
     """
     LDAP search helper method for finding groups
     LDAP search helper method for finding groups
 
 
@@ -418,7 +426,7 @@ class LdapConnection(object):
     else:
     else:
       return []
       return []
 
 
-  def find_members_of_group(self, dn, search_attr, ldap_filter, scope=ldap.SCOPE_SUBTREE):
+  def find_members_of_group(self, dn, search_attr, ldap_filter, scope=SCOPE_SUBTREE):
     if ldap_filter and not ldap_filter.startswith('('):
     if ldap_filter and not ldap_filter.startswith('('):
       ldap_filter = '(' + ldap_filter + ')'
       ldap_filter = '(' + ldap_filter + ')'
 
 

+ 4 - 1
apps/useradmin/src/useradmin/views.py

@@ -27,7 +27,10 @@ import json
 from axes.decorators import FAILURE_LIMIT, LOCK_OUT_AT_FAILURE
 from axes.decorators import FAILURE_LIMIT, LOCK_OUT_AT_FAILURE
 from axes.models import AccessAttempt
 from axes.models import AccessAttempt
 from axes.utils import reset
 from axes.utils import reset
-import ldap
+try:
+  import ldap
+except ImportError:
+  LOG.warn('ldap module not found')
 
 
 from django.urls import reverse
 from django.urls import reverse
 from django.forms import ValidationError
 from django.forms import ValidationError

+ 3 - 2
desktop/core/requirements.txt

@@ -35,13 +35,14 @@ lockfile==0.12.2
 lxml==4.5.0
 lxml==4.5.0
 Mako==1.0.7
 Mako==1.0.7
 Markdown==3.1
 Markdown==3.1
-MarkupSafe==0.9.3
+#MarkupSafe==0.9.3
 nose==1.3.7
 nose==1.3.7
 openpyxl==2.6.2
 openpyxl==2.6.2
 pyformance==0.3.2
 pyformance==0.3.2
 pysaml2>=4.5.0
 pysaml2>=4.5.0
 python-crontab==2.3.6
 python-crontab==2.3.6
 python-dateutil==2.4.2
 python-dateutil==2.4.2
+python-daemon==2.2.4
 # python-ldap==3.1.0
 # python-ldap==3.1.0
 python-oauth2==1.1.0
 python-oauth2==1.1.0
 pytidylib==0.3.2
 pytidylib==0.3.2
@@ -52,7 +53,7 @@ requests-kerberos==0.12.0
 rsa==3.4.2
 rsa==3.4.2
 sasl==0.2.1  # Move to https://pypi.org/project/sasl3/ ?
 sasl==0.2.1  # Move to https://pypi.org/project/sasl3/ ?
 six==1.12.0
 six==1.12.0
-SQLAlchemy==1.3.15
+SQLAlchemy==1.3.8
 sqlparse==0.2.0
 sqlparse==0.2.0
 tablib==0.13.0
 tablib==0.13.0
 thrift==0.13.0
 thrift==0.13.0

+ 23 - 9
desktop/core/src/desktop/auth/backend.py

@@ -31,9 +31,17 @@ User to remain a django.contrib.auth.models.User object.
 from builtins import object
 from builtins import object
 from importlib import import_module
 from importlib import import_module
 
 
-import ldap
 import logging
 import logging
-import pam
+LOG = logging.getLogger(__name__)
+
+try:
+  import ldap
+except ImportError:
+  LOG.warn('ldap module not found')
+try:
+  import pam
+except ImportError:
+  LOG.warn('pam module not found')
 import requests
 import requests
 
 
 import django.contrib.auth.backends
 import django.contrib.auth.backends
@@ -42,11 +50,20 @@ from django.core.urlresolvers import reverse
 from django.core.exceptions import ImproperlyConfigured, PermissionDenied
 from django.core.exceptions import ImproperlyConfigured, PermissionDenied
 from django.http import HttpResponseRedirect
 from django.http import HttpResponseRedirect
 from django.forms import ValidationError
 from django.forms import ValidationError
-from django_auth_ldap.backend import LDAPBackend
-from django_auth_ldap.config import LDAPSearch
+try:
+  from django_auth_ldap.backend import LDAPBackend
+  from django_auth_ldap.config import LDAPSearch
+except ImportError:
+  LOG.warn('django_auth_ldap module not found')
+  class LDAPSearch: pass
+  class LDAPSearch: pass
 from liboauth.metrics import oauth_authentication_time
 from liboauth.metrics import oauth_authentication_time
-from mozilla_django_oidc.auth import OIDCAuthenticationBackend, default_username_algo
-from mozilla_django_oidc.utils import absolutify, import_from_settings
+try:
+  from mozilla_django_oidc.auth import OIDCAuthenticationBackend, default_username_algo
+  from mozilla_django_oidc.utils import absolutify, import_from_settings
+except ImportError:
+  LOG.warn('mozilla_django_oidc module not found')
+  class OIDCAuthenticationBackend: pass
 
 
 from desktop import metrics
 from desktop import metrics
 from desktop.conf import AUTH, LDAP, OIDC, ENABLE_ORGANIZATIONS
 from desktop.conf import AUTH, LDAP, OIDC, ENABLE_ORGANIZATIONS
@@ -57,9 +74,6 @@ from useradmin.models import get_profile, get_default_user_group, UserProfile, U
 from useradmin.organization import get_organization
 from useradmin.organization import get_organization
 
 
 
 
-LOG = logging.getLogger(__name__)
-
-
 # TODO: slowly move those utils to the useradmin module
 # TODO: slowly move those utils to the useradmin module
 
 
 def load_augmentation_class():
 def load_augmentation_class():

+ 6 - 5
desktop/core/src/desktop/lib/fs/gc/client.py

@@ -15,9 +15,13 @@
 # limitations under the License.
 # limitations under the License.
 from __future__ import absolute_import
 from __future__ import absolute_import
 
 
-
 import logging
 import logging
-import gcs_oauth2_boto_plugin
+LOG = logging.getLogger(__name__)
+
+try:
+  import gcs_oauth2_boto_plugin
+except ImportError:
+  LOG.warn('gcs_oauth2_boto_plugin module not found')
 import json
 import json
 
 
 from aws.s3.s3fs import S3FileSystem
 from aws.s3.s3fs import S3FileSystem
@@ -32,9 +36,6 @@ from desktop.lib.idbroker import conf as conf_idbroker
 from desktop.lib.idbroker.client import IDBroker
 from desktop.lib.idbroker.client import IDBroker
 
 
 
 
-LOG = logging.getLogger(__name__)
-
-
 def get_credential_provider(config, user):
 def get_credential_provider(config, user):
   return CredentialProviderIDBroker(IDBroker.from_core_site('gs', user)) if conf_idbroker.is_idbroker_enabled('gs') else \
   return CredentialProviderIDBroker(IDBroker.from_core_site('gs', user)) if conf_idbroker.is_idbroker_enabled('gs') else \
       CredentialProviderConf(config)
       CredentialProviderConf(config)

+ 33 - 14
tools/docker/hue/Dockerfile.py3

@@ -8,40 +8,59 @@ LABEL description="Hue SQL Assistant - gethue.com"
 RUN export PYTHON_VER=python3.6
 RUN export PYTHON_VER=python3.6
 
 
 RUN apt-get update -y && apt-get install -y \
 RUN apt-get update -y && apt-get install -y \
-  python3-pip  #libmariadb-dev-compat  # python3.6-dev \
+  python3-pip \
   libkrb5-dev  \
   libkrb5-dev  \
   libsasl2-modules-gssapi-mit \
   libsasl2-modules-gssapi-mit \
   libsasl2-dev \
   libsasl2-dev \
   libkrb5-dev \
   libkrb5-dev \
   libxml2-dev \
   libxml2-dev \
-  libxslt-dev \ #libssl-dev \
+  libxslt-dev \
   libmysqlclient-dev \
   libmysqlclient-dev \
   libldap2-dev \
   libldap2-dev \
   libsnappy-dev \
   libsnappy-dev \
-  python3.6-venv
+  python3.6-venv \
+  rsync \
+  curl \
+  sudo \
+  git
+
+#libmariadb-dev-compat  # python3.6-dev #libssl-dev
+
+RUN pip3 install --upgrade setuptools
+RUN pip3 install virtualenv
+
+# Need recent version for Ubuntu
+RUN curl -sL https://deb.nodesource.com/setup_10.x | sudo bash - \
+  && apt-get install -y nodejs
+
+RUN addgroup hue && useradd -r -u 1001 -g hue hue
 
 
 ADD . /hue
 ADD . /hue
 WORKDIR /hue
 WORKDIR /hue
 
 
-#RUN pip3 install virtualenv
+RUN chown -R hue /hue
+
+RUN mkdir /hue/build && chown -R hue /hue/build && mkdir /usr/share/hue && chown -R hue /usr/share/hue
+
+# Not doing a `make prod`, so manually getting production ini
+RUN rm desktop/conf/*
+COPY desktop/conf.dist desktop/conf
+
+#USER hue
+
 #RUN python3.6 -m venv python_env
 #RUN python3.6 -m venv python_env
 
 
 #SHELL ["/bin/bash", "-c"]
 #SHELL ["/bin/bash", "-c"]
 #RUN source python_env/bin/activate
 #RUN source python_env/bin/activate
 
 
-RUN pip3 install -r desktop/core/requirements.txt
-
+RUN rm -r desktop/core/ext-py
+# RUN pip3 install -r desktop/core/requirements.txt
 
 
-# Not doing a `make prod`, so manually getting production ini
-RUN rm desktop/conf/*
-COPY desktop/conf.dist desktop/conf
 
 
-# Need recent version for Ubuntu
-# RUN curl -sL https://deb.nodesource.com/setup_10.x | sudo bash - \
-#  && apt-get install -y nodejs
+RUN PREFIX=/usr/share PYTHON_VER=python3.6 make install
+RUN chown -R hue /usr/share/hue
+#RUN useradd -ms /bin/bash hue && chown -R hue /usr/share/hue
 
 
-RUN PREFIX=/usr/share PYTHON_VER=python3.6 make install # install not working?
-RUN useradd -ms /bin/bash hue && chown -R hue /usr/share/hue
 
 
 # Only keep install dir
 # Only keep install dir
 # Note: get more minimal image by pulling install dir in a stage 2 image
 # Note: get more minimal image by pulling install dir in a stage 2 image