Просмотр исходного кода

HUE-8737 [py3] Workaround snappy and avro modules missing

Romain 6 лет назад
Родитель
Сommit
b9aa258dc9

+ 12 - 9
apps/filebrowser/src/filebrowser/views.py

@@ -24,7 +24,6 @@ import logging
 import mimetypes
 import operator
 import os
-import parquet
 import posixpath
 import re
 import stat as stat_module
@@ -50,8 +49,8 @@ from django.utils.html import escape
 from django.utils.translation import ugettext as _
 
 from aws.s3.s3fs import S3FileSystemException, S3ListAllBucketsException
-from avro import datafile, io
 from desktop import appmanager
+from desktop.auth.backend import is_admin
 from desktop.lib import i18n
 from desktop.lib.conf import coerce_bool
 from desktop.lib.django_util import render, format_preserving_redirect
@@ -79,16 +78,18 @@ from filebrowser.forms import RenameForm, UploadFileForm, UploadArchiveForm, MkD
                               TrashPurgeForm, SetReplicationFactorForm
 
 
-from desktop.auth.backend import is_admin
-
 if sys.version_info[0] > 2:
-  from io import string_io as string_io
+  import io
+  from io import StringIO as string_io
   from urllib.parse import quote as urllib_quote
   from urllib.parse import unquote as urllib_unquote
 else:
   from cStringIO import StringIO as string_io
   from urllib import quote as urllib_quote
   from urllib import unquote as urllib_unquote
+  import parquet
+  from avro import datafile, io
+
 
 DEFAULT_CHUNK_SIZE_BYTES = 1024 * 4 # 4KB
 MAX_CHUNK_SIZE_BYTES = 1024 * 1024 # 1MB
@@ -101,9 +102,11 @@ BYTES_PER_SENTENCE = 2
 # The maximum size the file editor will allow you to edit
 MAX_FILEEDITOR_SIZE = 256 * 1024
 
-INLINE_DISPLAY_MIMETYPE = re.compile('video/|image/|audio/|application/pdf|application/msword|application/excel|'
-                                     'application/vnd\.ms|'
-                                     'application/vnd\.openxmlformats')
+INLINE_DISPLAY_MIMETYPE = re.compile(
+    'video/|image/|audio/|application/pdf|application/msword|application/excel|'
+    'application/vnd\.ms|'
+    'application/vnd\.openxmlformats'
+)
 
 INLINE_DISPLAY_MIMETYPE_EXCEPTIONS = re.compile('image/svg\+xml')
 
@@ -1345,7 +1348,7 @@ def _upload_file(request):
           'result': _massage_stats(request, stat_absolute_path(filepath, request.fs.stats(filepath))),
           'next': request.GET.get("next")
         })
- 
+
         return response
     else:
         raise PopupException(_("Error in upload form: %s") % (form.errors,))

+ 3 - 4
apps/hbase/src/hbase/views.py

@@ -24,10 +24,9 @@ import re
 import sys
 import urllib.request, urllib.parse, urllib.error
 
-from avro import datafile, io
-
 from django.utils.translation import ugettext as _
 
+from desktop.auth.backend import is_admin
 from desktop.lib.django_util import JsonResponse, render
 
 from hbase import conf
@@ -37,12 +36,12 @@ from hbase.api import HbaseApi
 from hbase.management.commands import hbase_setup
 from hbase.server.hbase_lib import get_thrift_type
 
-from desktop.auth.backend import is_admin
-
 if sys.version_info[0] > 2:
   from io import StringIO as string_io
 else:
   from cStringIO import StringIO as string_io
+  from avro import datafile, io
+
 
 LOG = logging.getLogger(__name__)
 

+ 6 - 6
apps/zookeeper/src/zookeeper/stats.py

@@ -97,12 +97,12 @@ class ZooKeeperStats(object):
         s.settimeout(self._timeout)
         data = ""
         try:
-	    s.connect(self._address)
-	    s.send(cmd)
-	    data = s.recv(2048)
-	    s.close()
+            s.connect(self._address)
+            s.send(cmd)
+            data = s.recv(2048)
+            s.close()
         except Exception as e:
-	    LOG.error('Problem connecting to host %s, exception raised : %s' % (self._host, e))
+	          LOG.error('Problem connecting to host %s, exception raised : %s' % (self._host, e))
         return data
 
     def _parse(self, data):
@@ -124,7 +124,7 @@ class ZooKeeperStats(object):
 
         result = {}
         if not data:
-	    return result
+	          return result
         h = string_io(data)
 
         version = h.readline()

+ 14 - 14
desktop/core/src/desktop/migrations/0001_initial.py

@@ -24,11 +24,11 @@ class Migration(migrations.Migration):
             name='Document',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('name', models.CharField(default=b'', max_length=255)),
-                ('description', models.TextField(default=b'')),
+                ('name', models.CharField(default='', max_length=255)),
+                ('description', models.TextField(default='')),
                 ('last_modified', models.DateTimeField(auto_now=True, db_index=True, verbose_name='Last modified')),
                 ('version', models.SmallIntegerField(default=1, verbose_name='Schema version')),
-                ('extra', models.TextField(default=b'')),
+                ('extra', models.TextField(default='')),
                 ('object_id', models.PositiveIntegerField()),
                 ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
             ],
@@ -37,12 +37,12 @@ class Migration(migrations.Migration):
             name='Document2',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('name', models.CharField(default=b'', max_length=255)),
-                ('description', models.TextField(default=b'')),
+                ('name', models.CharField(default='', max_length=255)),
+                ('description', models.TextField(default='')),
                 ('uuid', models.CharField(db_index=True, default=desktop.models.uuid_default, max_length=36)),
-                ('type', models.CharField(db_index=True, default=b'', help_text='Type of document, e.g. Hive query, Oozie workflow, Search Dashboard...', max_length=32)),
-                ('data', models.TextField(default=b'{}')),
-                ('extra', models.TextField(default=b'')),
+                ('type', models.CharField(db_index=True, default='', help_text='Type of document, e.g. Hive query, Oozie workflow, Search Dashboard...', max_length=32)),
+                ('data', models.TextField(default='{}')),
+                ('extra', models.TextField(default='')),
                 ('last_modified', models.DateTimeField(auto_now=True, db_index=True, verbose_name='Time last modified')),
                 ('version', models.SmallIntegerField(db_index=True, default=1, verbose_name='Document version')),
                 ('is_history', models.BooleanField(db_index=True, default=False)),
@@ -56,18 +56,18 @@ class Migration(migrations.Migration):
             name='Document2Permission',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('perms', models.CharField(choices=[(b'read', b'read'), (b'write', b'write'), (b'comment', b'comment')], db_index=True, default=b'read', max_length=10)),
+                ('perms', models.CharField(choices=[('read', 'read'), ('write', 'write'), ('comment', 'comment')], db_index=True, default='read', max_length=10)),
                 ('doc', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='desktop.Document2')),
-                ('groups', models.ManyToManyField(db_index=True, db_table=b'documentpermission2_groups', to='auth.Group')),
+                ('groups', models.ManyToManyField(db_index=True, db_table='documentpermission2_groups', to='auth.Group')),
             ],
         ),
         migrations.CreateModel(
             name='DocumentPermission',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('perms', models.CharField(choices=[(b'read', b'read'), (b'write', b'write')], default=b'read', max_length=10)),
+                ('perms', models.CharField(choices=[('read', 'read'), ('write', 'write')], default='read', max_length=10)),
                 ('doc', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='desktop.Document')),
-                ('groups', models.ManyToManyField(db_index=True, db_table=b'documentpermission_groups', to='auth.Group')),
+                ('groups', models.ManyToManyField(db_index=True, db_table='documentpermission_groups', to='auth.Group')),
             ],
         ),
         migrations.CreateModel(
@@ -106,12 +106,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='documentpermission',
             name='users',
-            field=models.ManyToManyField(db_index=True, db_table=b'documentpermission_users', to=settings.AUTH_USER_MODEL),
+            field=models.ManyToManyField(db_index=True, db_table='documentpermission_users', to=settings.AUTH_USER_MODEL),
         ),
         migrations.AddField(
             model_name='document2permission',
             name='users',
-            field=models.ManyToManyField(db_index=True, db_table=b'documentpermission2_users', to=settings.AUTH_USER_MODEL),
+            field=models.ManyToManyField(db_index=True, db_table='documentpermission2_users', to=settings.AUTH_USER_MODEL),
         ),
         migrations.AddField(
             model_name='document2',

+ 2 - 1
desktop/libs/hadoop/src/hadoop/hdfs_site.py

@@ -24,6 +24,7 @@ from . import conf
 from . import confparse
 from hadoop.conf import DEFAULT_NN_HTTP_PORT
 
+
 LOG = logging.getLogger(__name__)
 
 _HDFS_SITE_DICT = None
@@ -82,7 +83,7 @@ def _parse_hdfs_site():
 
   try:
     hdfs_site_path = os.path.join(conf.HDFS_CLUSTERS['default'].HADOOP_CONF_DIR.get(), 'hdfs-site.xml')
-    data = file(hdfs_site_path, 'r').read()
+    data = open(hdfs_site_path, 'r').read()
   except KeyError:
     data = ""
   except IOError as err:

+ 2 - 1
desktop/libs/indexer/src/indexer/api3.py

@@ -56,13 +56,14 @@ from indexer.solr_client import SolrClient, MAX_UPLOAD_SIZE
 from indexer.indexers.flume import FlumeIndexer
 
 if sys.version_info[0] > 2:
-  from io import string_io as string_io
+  from io import StringIO as string_io
   from urllib.parse import urlparse, unquote as urllib_unquote
 else:
   from StringIO import StringIO as string_io
   from urllib import unquote as urllib_unquote
   from urlparse import urlparse
 
+
 LOG = logging.getLogger(__name__)
 
 

+ 1 - 1
desktop/libs/metadata/src/metadata/optimizer_client.py

@@ -47,7 +47,7 @@ OPTIMIZER_TENANT_ID_CACHE_KEY = 'navopt-tenant-id'
 try:
   from navoptapi.api_lib import ApiLib
 except Exception as e:
-  LOG.exception('NavOpt module is not installed: %s' % e)
+  LOG.warn('NavOpt module is not installed: %s' % e)
 
 
 class NavOptException(Exception):

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/altus.py

@@ -36,7 +36,7 @@ DATE_FORMAT = "%Y-%m-%d"
 try:
   from navoptapi.api_lib import ApiLib
 except Exception as e:
-  LOG.exception('NavOpt module is not installed: %s' % e)
+  LOG.warn('NavOpt module is not installed: %s' % e)
 
 
 def _exec(service, command, parameters=None):