Browse Source

HUE-8737 [py3] Make a lot of non essnential modules optional

Romain 5 years ago
parent
commit
e4853b6f1e

+ 7 - 3
Makefile.vars

@@ -42,7 +42,7 @@ VER_ERR_MSG = "Variable PYTHON_VER is $(PYTHON_VER) but it only supports python2
 PYTHON_VER ?= python2.7
 PYTHON_EXE = $(shell echo $(PYTHON_VER) | sed "s/\.//")
 
-$(info "PYTHON_VER is $(PYTHON_VER)")
+$(info "PYTHON_VER is $(PYTHON_VER).")
 
 ifeq ($(PYTHON_VER),python2.7)
   $(info "Python 2 module install of desktop/ext-py")
@@ -71,6 +71,8 @@ endif
 
 ifeq ($(SYS_PYTHON),)
   $(error "Error: Need python version 2.7 or >= 3.5")
+else
+  $(info "SYS_PYTHON is $(SYS_PYTHON).")
 endif
 
 HADOOP_HOME ?= /usr/lib/hadoop
@@ -94,6 +96,8 @@ STATIC_DIR := $(BLD_DIR)/static
 ENV_PYTHON := $(BLD_DIR_ENV)/bin/$(notdir $(SYS_PYTHON))
 ENV_PYTHON_VERSION = $(shell $(ENV_PYTHON) -c 'import sys; print ("python%d.%d"% sys.version_info[:2])')
 
+$(info "ENV_PYTHON is $(ENV_PYTHON).")
+
 
 ##############################
 # ENV_EASY_INSTALL uses the easy_install script installed in the virtual
@@ -107,7 +111,7 @@ ENV_PIP := $(ENV_PYTHON) $(BLD_DIR_BIN)/pip
 ##############################
 # This version is substituted through to the tarballs and packages.
 ##############################
-DESKTOP_VERSION := $(shell python <(cat $(ROOT)/VERSION; echo print '(VERSION)'))
+DESKTOP_VERSION := $(shell $(SYS_PYTHON) <(cat $(ROOT)/VERSION; echo print '(VERSION)'))
 MAVEN_VERSION = $(DESKTOP_VERSION)-SNAPSHOT
 
 
@@ -124,5 +128,5 @@ PYBABEL := $(ROOT)/build/env/bin/pybabel
 ##############################
 # Path to files for pip requirements
 ##############################
-REQUIREMENT_FILE := $(ROOT)/desktop/core/requirements_py3.txt
+REQUIREMENT_FILE := $(ROOT)/desktop/core/requirements.txt
 NAVOPTAPI_WHL := $(ROOT)/desktop/core/wheels/navoptapi-1.0.0-py3-none-any.whl

+ 18 - 10
apps/useradmin/src/useradmin/ldap_access.py

@@ -18,20 +18,28 @@
 This module provides access to LDAP servers, along with some basic functionality required for Hue and
 User Admin to work seamlessly with LDAP.
 """
-from builtins import str
-from builtins import object
-import ldap
-import ldap.filter
+from builtins import str, object
+
 import logging
+
+LOG = logging.getLogger(__name__)
+
+try:
+  import ldap
+  import ldap.filter
+  from ldap import SCOPE_SUBTREE
+except ImportError:
+  LOG.warn('ldap module not found')
+  SCOPE_SUBTREE = None
 import re
 
+from django.utils.encoding import smart_text
+
 import desktop.conf
 from desktop.lib.python_util import CaseInsensitiveDict
-from useradmin.models import User
-from django.utils.encoding import smart_text
 
+from useradmin.models import User
 
-LOG = logging.getLogger(__name__)
 CACHED_LDAP_CONN = None
 
 
@@ -301,7 +309,7 @@ class LdapConnection(object):
 
     return group_info
 
-  def find_users(self, username_pattern, search_attr=None, user_name_attr=None, user_filter=None, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
+  def find_users(self, username_pattern, search_attr=None, user_name_attr=None, user_filter=None, find_by_dn=False, scope=SCOPE_SUBTREE):
     """
     LDAP search helper method finding users. This supports searching for users
     by distinguished name, or the configured username attribute.
@@ -364,7 +372,7 @@ class LdapConnection(object):
 
     return None
 
-  def find_groups(self, groupname_pattern, search_attr=None, group_name_attr=None, group_member_attr=None, group_filter=None, find_by_dn=False, scope=ldap.SCOPE_SUBTREE):
+  def find_groups(self, groupname_pattern, search_attr=None, group_name_attr=None, group_member_attr=None, group_filter=None, find_by_dn=False, scope=SCOPE_SUBTREE):
     """
     LDAP search helper method for finding groups
 
@@ -418,7 +426,7 @@ class LdapConnection(object):
     else:
       return []
 
-  def find_members_of_group(self, dn, search_attr, ldap_filter, scope=ldap.SCOPE_SUBTREE):
+  def find_members_of_group(self, dn, search_attr, ldap_filter, scope=SCOPE_SUBTREE):
     if ldap_filter and not ldap_filter.startswith('('):
       ldap_filter = '(' + ldap_filter + ')'
 

+ 4 - 1
apps/useradmin/src/useradmin/views.py

@@ -27,7 +27,10 @@ import json
 from axes.decorators import FAILURE_LIMIT, LOCK_OUT_AT_FAILURE
 from axes.models import AccessAttempt
 from axes.utils import reset
-import ldap
+try:
+  import ldap
+except ImportError:
+  LOG.warn('ldap module not found')
 
 from django.urls import reverse
 from django.forms import ValidationError

+ 3 - 2
desktop/core/requirements.txt

@@ -35,13 +35,14 @@ lockfile==0.12.2
 lxml==4.5.0
 Mako==1.0.7
 Markdown==3.1
-MarkupSafe==0.9.3
+#MarkupSafe==0.9.3
 nose==1.3.7
 openpyxl==2.6.2
 pyformance==0.3.2
 pysaml2>=4.5.0
 python-crontab==2.3.6
 python-dateutil==2.4.2
+python-daemon==2.2.4
 # python-ldap==3.1.0
 python-oauth2==1.1.0
 pytidylib==0.3.2
@@ -52,7 +53,7 @@ requests-kerberos==0.12.0
 rsa==3.4.2
 sasl==0.2.1  # Move to https://pypi.org/project/sasl3/ ?
 six==1.12.0
-SQLAlchemy==1.3.15
+SQLAlchemy==1.3.8
 sqlparse==0.2.0
 tablib==0.13.0
 thrift==0.13.0

+ 23 - 9
desktop/core/src/desktop/auth/backend.py

@@ -31,9 +31,17 @@ User to remain a django.contrib.auth.models.User object.
 from builtins import object
 from importlib import import_module
 
-import ldap
 import logging
-import pam
+LOG = logging.getLogger(__name__)
+
+try:
+  import ldap
+except ImportError:
+  LOG.warn('ldap module not found')
+try:
+  import pam
+except ImportError:
+  LOG.warn('pam module not found')
 import requests
 
 import django.contrib.auth.backends
@@ -42,11 +50,20 @@ from django.core.urlresolvers import reverse
 from django.core.exceptions import ImproperlyConfigured, PermissionDenied
 from django.http import HttpResponseRedirect
 from django.forms import ValidationError
-from django_auth_ldap.backend import LDAPBackend
-from django_auth_ldap.config import LDAPSearch
+try:
+  from django_auth_ldap.backend import LDAPBackend
+  from django_auth_ldap.config import LDAPSearch
+except ImportError:
+  LOG.warn('django_auth_ldap module not found')
+  class LDAPSearch: pass
+  class LDAPSearch: pass
 from liboauth.metrics import oauth_authentication_time
-from mozilla_django_oidc.auth import OIDCAuthenticationBackend, default_username_algo
-from mozilla_django_oidc.utils import absolutify, import_from_settings
+try:
+  from mozilla_django_oidc.auth import OIDCAuthenticationBackend, default_username_algo
+  from mozilla_django_oidc.utils import absolutify, import_from_settings
+except ImportError:
+  LOG.warn('mozilla_django_oidc module not found')
+  class OIDCAuthenticationBackend: pass
 
 from desktop import metrics
 from desktop.conf import AUTH, LDAP, OIDC, ENABLE_ORGANIZATIONS
@@ -57,9 +74,6 @@ from useradmin.models import get_profile, get_default_user_group, UserProfile, U
 from useradmin.organization import get_organization
 
 
-LOG = logging.getLogger(__name__)
-
-
 # TODO: slowly move those utils to the useradmin module
 
 def load_augmentation_class():

+ 6 - 5
desktop/core/src/desktop/lib/fs/gc/client.py

@@ -15,9 +15,13 @@
 # limitations under the License.
 from __future__ import absolute_import
 
-
 import logging
-import gcs_oauth2_boto_plugin
+LOG = logging.getLogger(__name__)
+
+try:
+  import gcs_oauth2_boto_plugin
+except ImportError:
+  LOG.warn('gcs_oauth2_boto_plugin module not found')
 import json
 
 from aws.s3.s3fs import S3FileSystem
@@ -32,9 +36,6 @@ from desktop.lib.idbroker import conf as conf_idbroker
 from desktop.lib.idbroker.client import IDBroker
 
 
-LOG = logging.getLogger(__name__)
-
-
 def get_credential_provider(config, user):
   return CredentialProviderIDBroker(IDBroker.from_core_site('gs', user)) if conf_idbroker.is_idbroker_enabled('gs') else \
       CredentialProviderConf(config)

+ 33 - 14
tools/docker/hue/Dockerfile.py3

@@ -8,40 +8,59 @@ LABEL description="Hue SQL Assistant - gethue.com"
 RUN export PYTHON_VER=python3.6
 
 RUN apt-get update -y && apt-get install -y \
-  python3-pip  #libmariadb-dev-compat  # python3.6-dev \
+  python3-pip \
   libkrb5-dev  \
   libsasl2-modules-gssapi-mit \
   libsasl2-dev \
   libkrb5-dev \
   libxml2-dev \
-  libxslt-dev \ #libssl-dev \
+  libxslt-dev \
   libmysqlclient-dev \
   libldap2-dev \
   libsnappy-dev \
-  python3.6-venv
+  python3.6-venv \
+  rsync \
+  curl \
+  sudo \
+  git
+
+#libmariadb-dev-compat  # python3.6-dev #libssl-dev
+
+RUN pip3 install --upgrade setuptools
+RUN pip3 install virtualenv
+
+# Need recent version for Ubuntu
+RUN curl -sL https://deb.nodesource.com/setup_10.x | sudo bash - \
+  && apt-get install -y nodejs
+
+RUN addgroup hue && useradd -r -u 1001 -g hue hue
 
 ADD . /hue
 WORKDIR /hue
 
-#RUN pip3 install virtualenv
+RUN chown -R hue /hue
+
+RUN mkdir /hue/build && chown -R hue /hue/build && mkdir /usr/share/hue && chown -R hue /usr/share/hue
+
+# Not doing a `make prod`, so manually getting production ini
+RUN rm desktop/conf/*
+COPY desktop/conf.dist desktop/conf
+
+#USER hue
+
 #RUN python3.6 -m venv python_env
 
 #SHELL ["/bin/bash", "-c"]
 #RUN source python_env/bin/activate
 
-RUN pip3 install -r desktop/core/requirements.txt
-
+RUN rm -r desktop/core/ext-py
+# RUN pip3 install -r desktop/core/requirements.txt
 
-# Not doing a `make prod`, so manually getting production ini
-RUN rm desktop/conf/*
-COPY desktop/conf.dist desktop/conf
 
-# Need recent version for Ubuntu
-# RUN curl -sL https://deb.nodesource.com/setup_10.x | sudo bash - \
-#  && apt-get install -y nodejs
+RUN PREFIX=/usr/share PYTHON_VER=python3.6 make install
+RUN chown -R hue /usr/share/hue
+#RUN useradd -ms /bin/bash hue && chown -R hue /usr/share/hue
 
-RUN PREFIX=/usr/share PYTHON_VER=python3.6 make install # install not working?
-RUN useradd -ms /bin/bash hue && chown -R hue /usr/share/hue
 
 # Only keep install dir
 # Note: get more minimal image by pulling install dir in a stage 2 image