Explorar o código

HUE-8737 [core] Futurize desktop/core/src/desktop for Python 3.5

Ying Chen %!s(int64=6) %!d(string=hai) anos
pai
achega
4ff4fb110c
Modificáronse 100 ficheiros con 853 adicións e 506 borrados
  1. 12 12
      desktop/core/src/desktop/api.py
  2. 26 18
      desktop/core/src/desktop/api2.py
  3. 2 1
      desktop/core/src/desktop/api2_tests.py
  4. 3 2
      desktop/core/src/desktop/api_tests.py
  5. 3 2
      desktop/core/src/desktop/appmanager.py
  6. 5 4
      desktop/core/src/desktop/auth/backend.py
  7. 15 7
      desktop/core/src/desktop/auth/views.py
  8. 1 0
      desktop/core/src/desktop/auth/views_test.py
  9. 1 0
      desktop/core/src/desktop/celery.py
  10. 29 24
      desktop/core/src/desktop/conf.py
  11. 7 7
      desktop/core/src/desktop/configuration/api.py
  12. 1 0
      desktop/core/src/desktop/configuration/tests.py
  13. 1 0
      desktop/core/src/desktop/converter_tests.py
  14. 10 9
      desktop/core/src/desktop/converters.py
  15. 19 1
      desktop/core/src/desktop/lib/analytics/lib/hive.py
  16. 19 1
      desktop/core/src/desktop/lib/analytics/lib/impala.py
  17. 29 23
      desktop/core/src/desktop/lib/conf.py
  18. 12 4
      desktop/core/src/desktop/lib/conf_test.py
  19. 3 2
      desktop/core/src/desktop/lib/config_spec_dump.py
  20. 3 3
      desktop/core/src/desktop/lib/connectors/api.py
  21. 19 1
      desktop/core/src/desktop/lib/connectors/lib/hive.py
  22. 19 1
      desktop/core/src/desktop/lib/connectors/lib/impala.py
  23. 20 10
      desktop/core/src/desktop/lib/django_forms.py
  24. 2 2
      desktop/core/src/desktop/lib/django_mako.py
  25. 5 4
      desktop/core/src/desktop/lib/django_util.py
  26. 7 2
      desktop/core/src/desktop/lib/django_util_test.py
  27. 1 1
      desktop/core/src/desktop/lib/djangojson.py
  28. 3 1
      desktop/core/src/desktop/lib/djangothrift.py
  29. 5 3
      desktop/core/src/desktop/lib/djangothrift_test.py
  30. 11 3
      desktop/core/src/desktop/lib/export_csvxls.py
  31. 11 3
      desktop/core/src/desktop/lib/fs/__init__.py
  32. 13 5
      desktop/core/src/desktop/lib/fs/proxyfs.py
  33. 2 1
      desktop/core/src/desktop/lib/fs/proxyfs_test.py
  34. 5 3
      desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/TestService.py
  35. 2 1
      desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/constants.py
  36. 10 8
      desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/ttypes.py
  37. 1 0
      desktop/core/src/desktop/lib/idbroker/client.py
  38. 1 1
      desktop/core/src/desktop/lib/metrics/file_reporter.py
  39. 2 1
      desktop/core/src/desktop/lib/metrics/registry.py
  40. 4 3
      desktop/core/src/desktop/lib/parameterization.py
  41. 2 2
      desktop/core/src/desktop/lib/paths.py
  42. 2 0
      desktop/core/src/desktop/lib/python_util.py
  43. 1 0
      desktop/core/src/desktop/lib/python_util_test.py
  44. 13 6
      desktop/core/src/desktop/lib/rest/http_client.py
  45. 3 2
      desktop/core/src/desktop/lib/rest/http_client_test.py
  46. 6 3
      desktop/core/src/desktop/lib/rest/resource.py
  47. 1 1
      desktop/core/src/desktop/lib/scheduler/api.py
  48. 2 1
      desktop/core/src/desktop/lib/scheduler/lib/api.py
  49. 1 0
      desktop/core/src/desktop/lib/scheduler/lib/oozie.py
  50. 13 5
      desktop/core/src/desktop/lib/tasks/compress_files/compress_utils.py
  51. 7 5
      desktop/core/src/desktop/lib/tasks/extract_archive/extract_utils.py
  52. 8 2
      desktop/core/src/desktop/lib/test_export_csvxls.py
  53. 1 0
      desktop/core/src/desktop/lib/test_runners.py
  54. 1 0
      desktop/core/src/desktop/lib/test_utils.py
  55. 10 4
      desktop/core/src/desktop/lib/thread_util.py
  56. 10 3
      desktop/core/src/desktop/lib/thread_util_test.py
  57. 2 1
      desktop/core/src/desktop/lib/thrift_/TSSLSocketWithWildcardSAN.py
  58. 9 4
      desktop/core/src/desktop/lib/thrift_/http_client.py
  59. 13 6
      desktop/core/src/desktop/lib/thrift_sasl.py
  60. 32 22
      desktop/core/src/desktop/lib/thrift_util.py
  61. 6 4
      desktop/core/src/desktop/lib/thrift_util_test.py
  62. 1 0
      desktop/core/src/desktop/lib/vcs/apis/base_api.py
  63. 16 8
      desktop/core/src/desktop/lib/vcs/apis/github_readonly_api.py
  64. 15 7
      desktop/core/src/desktop/lib/vcs/github_client.py
  65. 3 1
      desktop/core/src/desktop/lib/view_util.py
  66. 3 1
      desktop/core/src/desktop/lib/view_util_test.py
  67. 60 47
      desktop/core/src/desktop/lib/wsgiserver.py
  68. 17 10
      desktop/core/src/desktop/log/__init__.py
  69. 4 2
      desktop/core/src/desktop/log/access.py
  70. 1 0
      desktop/core/src/desktop/log/log_buffer.py
  71. 2 1
      desktop/core/src/desktop/log/log_buffer_test.py
  72. 47 41
      desktop/core/src/desktop/manage_entry.py
  73. 10 9
      desktop/core/src/desktop/management/commands/config_dump.py
  74. 5 4
      desktop/core/src/desktop/management/commands/convert_documents.py
  75. 1 1
      desktop/core/src/desktop/management/commands/create_desktop_app.py
  76. 1 1
      desktop/core/src/desktop/management/commands/create_proxy_app.py
  77. 3 1
      desktop/core/src/desktop/management/commands/create_test_fs.py
  78. 2 2
      desktop/core/src/desktop/management/commands/create_user_directories.py
  79. 2 2
      desktop/core/src/desktop/management/commands/desktop_document_cleanup.py
  80. 3 1
      desktop/core/src/desktop/management/commands/generate_mdl.py
  81. 3 3
      desktop/core/src/desktop/management/commands/is_db_alive.py
  82. 1 0
      desktop/core/src/desktop/management/commands/run_windmill.py
  83. 3 2
      desktop/core/src/desktop/management/commands/runcherrypyserver.py
  84. 5 3
      desktop/core/src/desktop/management/commands/test.py
  85. 3 2
      desktop/core/src/desktop/management/commands/version.py
  86. 4 1
      desktop/core/src/desktop/metrics.py
  87. 12 12
      desktop/core/src/desktop/middleware.py
  88. 1 1
      desktop/core/src/desktop/middleware_test.py
  89. 43 32
      desktop/core/src/desktop/models.py
  90. 2 2
      desktop/core/src/desktop/old_migrations/0013_auto__add_unique_documenttag_owner_tag.py
  91. 2 2
      desktop/core/src/desktop/old_migrations/0014_auto__add_unique_document_content_type_object_id.py
  92. 2 2
      desktop/core/src/desktop/old_migrations/0015_auto__add_unique_documentpermission_doc_perms.py
  93. 1 1
      desktop/core/src/desktop/old_migrations/0016_auto__add_unique_document2_uuid_version_is_history.py
  94. 3 2
      desktop/core/src/desktop/redaction/engine.py
  95. 8 3
      desktop/core/src/desktop/redaction/tests.py
  96. 5 3
      desktop/core/src/desktop/settings.py
  97. 29 22
      desktop/core/src/desktop/supervisor.py
  98. 27 19
      desktop/core/src/desktop/tests.py
  99. 2 1
      desktop/core/src/desktop/tests_doc2.py
  100. 14 9
      desktop/core/src/desktop/views.py

+ 12 - 12
desktop/core/src/desktop/api.py

@@ -91,7 +91,7 @@ def massaged_tags_for_json(docs, user):
 
   ts['trash'] = massaged_tags(trash_tag, tag_doc_mapping)
   ts['history'] = massaged_tags(history_tag, tag_doc_mapping)
-  tags = list(set(tag_doc_mapping.keys() + [tag for tag in DocumentTag.objects.get_tags(user=user)])) # List of all personal and shared tags
+  tags = list(set(list(tag_doc_mapping.keys()) + [tag for tag in DocumentTag.objects.get_tags(user=user)])) # List of all personal and shared tags
 
   for tag in tags:
     massaged_tag = massaged_tags(tag, tag_doc_mapping)
@@ -104,7 +104,7 @@ def massaged_tags_for_json(docs, user):
     else:
       sharers[tag.owner].append(massaged_tag)
 
-  ts['notmine'] = [{'name': sharer.username, 'projects': projects} for sharer, projects in sharers.iteritems()]
+  ts['notmine'] = [{'name': sharer.username, 'projects': projects} for sharer, projects in sharers.items()]
   # Remove from my tags the trashed and history ones
   mine_filter = set(ts['trash']['docs'] + ts['history']['docs'])
   for tag in ts['mine']:
@@ -253,9 +253,9 @@ def add_tag(request):
       response['status'] = 0
     else:
       response['status'] = -1
-  except KeyError, e:
+  except KeyError as e:
     response['message'] = _('Form is missing %s field') % e.message
-  except Exception, e:
+  except Exception as e:
     response['message'] = force_unicode(e)
 
   return JsonResponse(response)
@@ -270,9 +270,9 @@ def tag(request):
     tag = DocumentTag.objects.tag(request.user, request_json['doc_id'], request_json.get('tag'), request_json.get('tag_id'))
     response['tag_id'] = tag.id
     response['status'] = 0
-  except KeyError, e:
+  except KeyError as e:
     response['message'] = _('Form is missing %s field') % e.message
-  except Exception, e:
+  except Exception as e:
     response['message'] = force_unicode(e)
 
   return JsonResponse(response)
@@ -287,9 +287,9 @@ def update_tags(request):
     doc = DocumentTag.objects.update_tags(request.user, request_json['doc_id'], request_json['tag_ids'])
     response['doc'] = massage_doc_for_json(doc, request.user)
     response['status'] = 0
-  except KeyError, e:
+  except KeyError as e:
     response['message'] = _('Form is missing %s field') % e.message
-  except Exception, e:
+  except Exception as e:
     response['message'] = force_unicode(e)
 
   return JsonResponse(response)
@@ -303,9 +303,9 @@ def remove_tag(request):
     DocumentTag.objects.delete_tag(request.POST.get('tag_id'), request.user)
     response['message'] = _('Project removed!')
     response['status'] = 0
-  except KeyError, e:
+  except KeyError as e:
     response['message'] = _('Form is missing %s field') % e.message
-  except Exception, e:
+  except Exception as e:
     response['message'] = force_unicode(e)
 
   return JsonResponse(response)
@@ -325,9 +325,9 @@ def update_permissions(request):
     response['message'] = _('Permissions updated!')
     response['status'] = 0
     response['doc'] = massage_doc_for_json(doc, request.user)
-  except KeyError, e:
+  except KeyError as e:
     response['message'] = _('Form is missing %s field') % e.message
-  except Exception, e:
+  except Exception as e:
     LOG.exception(e.message)
     response['message'] = force_unicode(e)
 

+ 26 - 18
desktop/core/src/desktop/api2.py

@@ -15,9 +15,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import map
+from builtins import str
 import logging
 import json
-import StringIO
+import sys
 import tempfile
 import zipfile
 
@@ -47,6 +51,10 @@ from desktop.lib.i18n import smart_str, force_unicode
 from desktop.models import Document2, Document, Directory, FilesystemException, uuid_default, \
   UserPreferences, get_user_preferences, set_user_preferences, get_cluster_config
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from StringIO import StringIO as string_io
 
 LOG = logging.getLogger(__name__)
 
@@ -57,7 +65,7 @@ def api_error_handler(func):
 
     try:
       return func(*args, **kwargs)
-    except Exception, e:
+    except Exception as e:
       LOG.exception('Error running %s' % func)
       response['status'] = -1
       response['message'] = force_unicode(str(e))
@@ -71,7 +79,7 @@ def api_error_handler(func):
 @api_error_handler
 def get_config(request):
   config = get_cluster_config(request.user)
-  config['clusters'] = get_clusters(request.user).values()
+  config['clusters'] = list(get_clusters(request.user).values())
   config['status'] = 0
 
   return JsonResponse(config)
@@ -85,7 +93,7 @@ def get_context_namespaces(request, interface):
   response = {}
   namespaces = []
 
-  clusters = get_clusters(request.user).values()
+  clusters = list(get_clusters(request.user).values())
 
   # Currently broken if not sent
   namespaces.extend([{
@@ -144,7 +152,7 @@ def get_context_computes(request, interface):
   response = {}
   computes = []
 
-  clusters = get_clusters(request.user).values()
+  clusters = list(get_clusters(request.user).values())
 
   if get_cluster_config(request.user)['has_computes']: # TODO: only based on interface selected?
     interpreter = get_interpreter(connector_type=interface, user=request.user)
@@ -187,7 +195,7 @@ def get_context_clusters(request, interface):
   response = {}
   clusters = []
 
-  cluster_configs = get_clusters(request.user).values()
+  cluster_configs = list(get_clusters(request.user).values())
 
   for cluster in cluster_configs:
     cluster = {
@@ -621,7 +629,7 @@ def share_document(request):
 
   doc = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
 
-  for name, perm in perms_dict.iteritems():
+  for name, perm in perms_dict.items():
     users = groups = None
     if perm.get('user_ids'):
       users = User.objects.in_bulk(perm.get('user_ids'))
@@ -667,7 +675,7 @@ def export_documents(request):
   else:
     filename = 'hue-documents-%s-(%s)' % (datetime.today().strftime('%Y-%m-%d'), num_docs)
 
-  f = StringIO.StringIO()
+  f = string_io()
 
   if doc_ids:
     doc_ids = ','.join(map(str, doc_ids))
@@ -683,7 +691,7 @@ def export_documents(request):
         try:
           from spark.models import Notebook
           zfile.writestr("notebook-%s-%s.txt" % (doc.name, doc.id), smart_str(Notebook(document=doc).get_str()))
-        except Exception, e:
+        except Exception as e:
           LOG.exception(e)
     zfile.close()
     response = HttpResponse(content_type="application/zip")
@@ -707,7 +715,7 @@ def import_documents(request):
       documents = json.loads(request.POST.get('documents'))
 
     documents = json.loads(documents)
-  except ValueError, e:
+  except ValueError as e:
     raise PopupException(_('Failed to import documents, the file does not contain valid JSON.'))
 
   # Validate documents
@@ -754,7 +762,7 @@ def import_documents(request):
   f.write(json.dumps(docs))
   f.flush()
 
-  stdout = StringIO.StringIO()
+  stdout = string_io()
   try:
     with transaction.atomic(): # We wrap both commands to commit loaddata & sync
       management.call_command('loaddata', f.name, verbosity=3, traceback=True, stdout=stdout, commit=False) # We need to use commit=False because commit=True will close the connection and make Document.objects.sync fail.
@@ -778,14 +786,14 @@ def import_documents(request):
             ('owner', doc['fields']['owner'][0])
           ]) for doc in docs]
       })
-  except Exception, e:
+  except Exception as e:
     LOG.error('Failed to run loaddata command in import_documents:\n %s' % stdout.getvalue())
     return JsonResponse({'status': -1, 'message': smart_str(e)})
   finally:
     stdout.close()
 
 def _update_imported_oozie_document(doc, uuids_map):
-  for key, value in uuids_map.iteritems():
+  for key, value in uuids_map.items():
     if value:
       doc['fields']['data'] = doc['fields']['data'].replace(key, value)
 
@@ -931,7 +939,7 @@ def _copy_document_with_owner(doc, owner, uuids_map):
   if doc['fields'].get('parent_directory'):
     parent_uuid = doc['fields']['parent_directory'][0]
 
-  if parent_uuid is not None and parent_uuid in uuids_map.keys():
+  if parent_uuid is not None and parent_uuid in list(uuids_map.keys()):
     if uuids_map[parent_uuid] is None:
       uuids_map[parent_uuid] = uuid_default()
     doc['fields']['parent_directory'] = [uuids_map[parent_uuid], 1, False]
@@ -944,7 +952,7 @@ def _copy_document_with_owner(doc, owner, uuids_map):
   # Remap dependencies if needed
   idx = 0
   for dep_uuid, dep_version, dep_is_history in doc['fields']['dependencies']:
-    if dep_uuid not in uuids_map.keys():
+    if dep_uuid not in list(uuids_map.keys()):
       LOG.warn('Could not find dependency UUID: %s in JSON import, may cause integrity errors if not found.' % dep_uuid)
     else:
       if uuids_map[dep_uuid] is None:
@@ -966,7 +974,7 @@ def _create_or_update_document_with_owner(doc, owner, uuids_map):
       doc['pk'] = existing_doc.pk
     else:
       create_new = True
-  except FilesystemException, e:
+  except FilesystemException as e:
     create_new = True
 
   if create_new:
@@ -977,7 +985,7 @@ def _create_or_update_document_with_owner(doc, owner, uuids_map):
   # Verify that parent exists, log warning and set parent to user's home directory if not found
   if doc['fields']['parent_directory']:
     uuid, version, is_history = doc['fields']['parent_directory']
-    if uuid not in uuids_map.keys() and \
+    if uuid not in list(uuids_map.keys()) and \
             not Document2.objects.filter(uuid=uuid, version=version, is_history=is_history).exists():
       LOG.warn('Could not find parent document with UUID: %s, will set parent to home directory' % uuid)
       doc['fields']['parent_directory'] = [home_dir.uuid, home_dir.version, home_dir.is_history]
@@ -987,7 +995,7 @@ def _create_or_update_document_with_owner(doc, owner, uuids_map):
   if doc['fields']['dependencies']:
     history_deps_list = []
     for index, (uuid, version, is_history) in enumerate(doc['fields']['dependencies']):
-      if not uuid in uuids_map.keys() and not is_history and \
+      if not uuid in list(uuids_map.keys()) and not is_history and \
               not Document2.objects.filter(uuid=uuid, version=version).exists():
           raise PopupException(_('Cannot import document, dependency with UUID: %s not found.') % uuid)
       elif is_history:

+ 2 - 1
desktop/core/src/desktop/api2_tests.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import json
 import re
 
@@ -28,7 +29,7 @@ from desktop.lib.test_utils import grant_access
 from desktop.models import Document2
 
 
-class TestApi2():
+class TestApi2(object):
 
   def setUp(self):
     self.client = make_logged_in_client(username="api2_user", groupname="default", recreate=True, is_superuser=False)

+ 3 - 2
desktop/core/src/desktop/api_tests.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import json
 
 from nose.tools import assert_true, assert_false, assert_equal, assert_not_equal, assert_raises
@@ -33,7 +34,7 @@ from pig.models import PigScript
 from useradmin.models import get_default_user_group
 
 
-class TestDocModelTags():
+class TestDocModelTags(object):
 
   def setUp(self):
     self.client = make_logged_in_client(username="tag_user", recreate=True, is_superuser=False)
@@ -195,7 +196,7 @@ class TestDocModelTags():
     # todo no default tag on test user?
 
 
-class TestDocModelPermissions():
+class TestDocModelPermissions(object):
 
   def setUp(self):
     self.client = make_logged_in_client(username="perm_user", groupname="default", recreate=True, is_superuser=False)

+ 3 - 2
desktop/core/src/desktop/appmanager.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import os
 import logging
 import re
@@ -52,7 +53,7 @@ def _import_module_or_none(module):
   try:
     __import__(module)
     return sys.modules[module]
-  except ImportError, ie:
+  except ImportError as ie:
     # If the exception came from us importing, we want to just
     # return None. We need to inspect the stack, though, so we properly
     # reraise in the case that the module we're importing triggered
@@ -204,7 +205,7 @@ class DesktopModuleInfo(object):
     return "DesktopModule(%s: %s)" % (self.nice_name, self.module.__name__)
 
 def get_apps(user):
-  return filter(lambda app: user.has_hue_permission(action="access", app=app.display_name), DESKTOP_APPS)
+  return [app for app in DESKTOP_APPS if user.has_hue_permission(action="access", app=app.display_name)]
 
 def get_apps_dict(user=None):
   if user is not None:

+ 5 - 4
desktop/core/src/desktop/auth/backend.py

@@ -28,6 +28,7 @@ Because Django's models are sometimes unfriendly, you'll want
 User to remain a django.contrib.auth.models.User object.
 """
 
+from builtins import object
 import ldap
 import logging
 import pam
@@ -106,7 +107,7 @@ def is_admin(user):
     try:
       user = rewrite_user(user)
       is_admin = user.has_hue_permission(action="superuser", app="useradmin")
-    except Exception, e:
+    except Exception as e:
       LOG.exception("Could not validate if %s is a superuser assuming False." % user)
       is_admin = False
   return is_admin
@@ -381,7 +382,7 @@ class LdapBackend(object):
               return User.objects.get_or_create(username=username)
           else:
             return User.objects.get_or_create(username=username)
-        except ValidationError, e:
+        except ValidationError as e:
           LOG.exception("LDAP username is invalid: %s" % username)
 
     self._backend = _LDAPBackend()
@@ -488,7 +489,7 @@ class LdapBackend(object):
       else:
         LOG.warn("%s not in an allowed login group" % username)
         return None
-    except ImproperlyConfigured, detail:
+    except ImproperlyConfigured as detail:
       LOG.warn("LDAP was not properly configured: %s", detail)
       return None
 
@@ -521,7 +522,7 @@ class LdapBackend(object):
       connection = ldap_access.get_connection_from_server(server)
       try:
         user_info = connection.find_users(username, find_by_dn=False)
-      except Exception, e:
+      except Exception as e:
         LOG.warn("Failed to find LDAP user: %s" % e)
 
       if not user_info:

+ 15 - 7
desktop/core/src/desktop/auth/views.py

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from future import standard_library
+standard_library.install_aliases()
 try:
   import oauth2 as oauth
 except:
@@ -22,7 +24,7 @@ except:
 
 import cgi
 import logging
-import urllib
+import sys
 from datetime import datetime
 
 from axes.decorators import watch_login
@@ -50,6 +52,12 @@ from useradmin.models import get_profile, UserProfile
 from useradmin.views import ensure_home_directory, require_change_password
 from notebook.connectors.base import get_api
 
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+  from urllib.parse import urlencode as urllib_urlencode
+else:
+  from urllib import urlencode as urllib_urlencode
+
 LOG = logging.getLogger(__name__)
 
 
@@ -134,7 +142,7 @@ def dt_login(request, from_modal=False):
 
         try:
           ensure_home_directory(request.fs, user)
-        except (IOError, WebHdfsException), e:
+        except (IOError, WebHdfsException) as e:
           LOG.error('Could not create home directory at login for %s.' % user, exc_info=e)
 
         if require_change_password(userprofile):
@@ -169,7 +177,7 @@ def dt_login(request, from_modal=False):
     if hasattr(request,'fs') and ('KnoxSpnegoDjangoBackend' in backend_names or 'SpnegoDjangoBackend' in backend_names or 'OIDCBackend' in backend_names or 'SAML2Backend' in backend_names) and request.user.is_authenticated():
       try:
         ensure_home_directory(request.fs, request.user)
-      except (IOError, WebHdfsException), e:
+      except (IOError, WebHdfsException) as e:
         LOG.error('Could not create home directory for %s user %s.' % ('OIDC' if 'OIDCBackend' in backend_names else 'SAML', request.user))
     if request.user.is_authenticated():
       return HttpResponseRedirect(redirect_to)
@@ -218,7 +226,7 @@ def dt_logout(request, next_page=None):
     session = {"type":session_app,"sourceMethod":"dt_logout"}
     try:
       get_api(request, session).close_session(session)
-    except Exception, e:
+    except Exception as e:
       LOG.warn("Error closing Impala session: %s" % e)
 
   backends = get_backends()
@@ -229,10 +237,10 @@ def dt_logout(request, next_page=None):
           response = backend.logout(request, next_page)
           if response:
             return response
-        except Exception, e:
+        except Exception as e:
           LOG.warn('Potential error on logout for user: %s with exception: %s' % (username, e))
 
-  if len(filter(lambda backend: hasattr(backend, 'logout'), backends)) == len(backends):
+  if len([backend for backend in backends if hasattr(backend, 'logout')]) == len(backends):
     LOG.warn("Failed to log out from all backends for user: %s" % (username))
 
   response = django.contrib.auth.views.logout(request, next_page)
@@ -263,7 +271,7 @@ def oauth_login(request):
 
   consumer = oauth.Consumer(OAUTH.CONSUMER_KEY.get(), OAUTH.CONSUMER_SECRET.get())
   client = oauth.Client(consumer)
-  resp, content = client.request(OAUTH.REQUEST_TOKEN_URL.get(), "POST", body=urllib.urlencode({
+  resp, content = client.request(OAUTH.REQUEST_TOKEN_URL.get(), "POST", body=urllib_urlencode({
                       'oauth_callback': 'http://' + request.get_host() + '/login/oauth_authenticated/'
                   }))
 

+ 1 - 0
desktop/core/src/desktop/auth/views_test.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import datetime
 from nose.tools import assert_true, assert_false, assert_equal
 

+ 1 - 0
desktop/core/src/desktop/celery.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 
 from __future__ import absolute_import, unicode_literals
+from __future__ import print_function
 
 import imp
 import os

+ 29 - 24
desktop/core/src/desktop/conf.py

@@ -22,6 +22,7 @@ import logging
 import os
 import socket
 import stat
+import sys
 
 from collections import OrderedDict
 
@@ -39,6 +40,10 @@ from desktop.lib.conf import Config, ConfigSection, UnspecifiedConfigSection,\
 from desktop.lib.i18n import force_unicode
 from desktop.lib.paths import get_desktop_root, get_run_root
 
+if sys.version_info[0] > 2:
+  from builtins import str as new_str
+else:
+  new_str = unicode
 
 LOG = logging.getLogger(__name__)
 
@@ -1862,28 +1867,28 @@ def validate_ldap(user, config):
       if bool(bind_dn) != bool(bind_password):
         if bind_dn == None:
           res.append((LDAP.BIND_DN,
-                    unicode(_("If you set bind_password, then you must set bind_dn."))))
+                    new_str(_("If you set bind_password, then you must set bind_dn."))))
         else:
           res.append((LDAP.BIND_PASSWORD,
-                      unicode(_("If you set bind_dn, then you must set bind_password."))))
+                      new_str(_("If you set bind_dn, then you must set bind_password."))))
   else:
     if config.NT_DOMAIN.get() is not None or \
         config.LDAP_USERNAME_PATTERN.get() is not None:
       if config.LDAP_URL.get() is None:
         res.append((config.LDAP_URL,
-                    unicode(_("LDAP is only partially configured. An LDAP URL must be provided."))))
+                    new_str(_("LDAP is only partially configured. An LDAP URL must be provided."))))
 
     if config.LDAP_URL.get() is not None:
       if config.NT_DOMAIN.get() is None and \
           config.LDAP_USERNAME_PATTERN.get() is None:
         res.append((config.LDAP_URL,
-                    unicode(_("LDAP is only partially configured. An NT Domain or username "
+                    new_str(_("LDAP is only partially configured. An NT Domain or username "
                     "search pattern must be provided."))))
 
     if config.LDAP_USERNAME_PATTERN.get() is not None and \
         '<username>' not in config.LDAP_USERNAME_PATTERN.get():
         res.append((config.LDAP_USERNAME_PATTERN,
-                   unicode(_("The LDAP username pattern should contain the special"
+                   new_str(_("The LDAP username pattern should contain the special"
                    "<username> replacement string for authentication."))))
 
   return res
@@ -1906,16 +1911,16 @@ def validate_database(user):
 
       # Promote InnoDB storage engine
       if innodb_table_count != total_table_count:
-        res.append(('PREFERRED_STORAGE_ENGINE', unicode(_('''We recommend MySQL InnoDB engine over
+        res.append(('PREFERRED_STORAGE_ENGINE', new_str(_('''We recommend MySQL InnoDB engine over
                                                       MyISAM which does not support transactions.'''))))
 
       if innodb_table_count != 0 and innodb_table_count != total_table_count:
-        res.append(('MYSQL_STORAGE_ENGINE', unicode(_('''All tables in the database must be of the same
+        res.append(('MYSQL_STORAGE_ENGINE', new_str(_('''All tables in the database must be of the same
                                                       storage engine type (preferably InnoDB).'''))))
-    except Exception, ex:
+    except Exception as ex:
       LOG.exception("Error in config validation of MYSQL_STORAGE_ENGINE: %s", ex)
   elif 'sqlite' in connection.vendor:
-    res.append(('SQLITE_NOT_FOR_PRODUCTION_USE', unicode(_('SQLite is only recommended for development environments. '
+    res.append(('SQLITE_NOT_FOR_PRODUCTION_USE', new_str(_('SQLite is only recommended for development environments. '
         'It might cause the "Database is locked" error. Migrating to MySQL, Oracle or PostgreSQL is strongly recommended.'))))
 
   # Check if django_migrations table is up to date
@@ -1936,7 +1941,7 @@ def validate_database(user):
               missing_migration_entries.append((app.name, migration_name))
 
     if missing_migration_entries:
-      res.append(('django_migrations', unicode(_('''django_migrations table seems to be corrupted or incomplete.
+      res.append(('django_migrations', new_str(_('''django_migrations table seems to be corrupted or incomplete.
                                                         %s entries are missing in the table: %s''') % (len(missing_migration_entries), missing_migration_entries))))
   except Exception:
     LOG.exception("Error in config validation of django_migrations")
@@ -1959,55 +1964,55 @@ def config_validator(user):
 
   doc_count = Document.objects.count()
   if doc_count > DOCUMENT2_MAX_ENTRIES:
-    res.append(('DOCUMENT_CLEANUP_WARNING', unicode(_('Desktop Document has more than %d entries: %d, '
+    res.append(('DOCUMENT_CLEANUP_WARNING', new_str(_('Desktop Document has more than %d entries: %d, '
                 'please run "hue desktop_document_cleanup --cm-managed" to remove old entries' % (DOCUMENT2_MAX_ENTRIES, doc_count)))))
 
   doc2_count = Document2.objects.count()
   if doc2_count > DOCUMENT2_MAX_ENTRIES:
-    res.append(('DOCUMENT2_CLEANUP_WARNING', unicode(_('Desktop Document2 has more than %d entries: %d, '
+    res.append(('DOCUMENT2_CLEANUP_WARNING', new_str(_('Desktop Document2 has more than %d entries: %d, '
                 'please run "hue desktop_document_cleanup --cm-managed" to remove old entries' % (DOCUMENT2_MAX_ENTRIES, doc2_count)))))
 
   session_count = Session.objects.count()
   if session_count > DOCUMENT2_MAX_ENTRIES:
-    res.append(('SESSION_CLEANUP_WARNING', unicode(_('Desktop Session has more than %d entries: %d, '
+    res.append(('SESSION_CLEANUP_WARNING', new_str(_('Desktop Session has more than %d entries: %d, '
                 'please run "hue desktop_document_cleanup --cm-managed" to remove old entries' % (DOCUMENT2_MAX_ENTRIES, session_count)))))
 
   qh_count = QueryHistory.objects.count()
   if qh_count > DOCUMENT2_MAX_ENTRIES:
-    res.append(('QueryHistory_CLEANUP_WARNING', unicode(_('Query History has more than %d entries: %d, '
+    res.append(('QueryHistory_CLEANUP_WARNING', new_str(_('Query History has more than %d entries: %d, '
                 'please run "hue desktop_document_cleanup --cm-managed" to remove old entries' % (DOCUMENT2_MAX_ENTRIES, qh_count)))))
 
   sq_count = SavedQuery.objects.count()
   if sq_count > DOCUMENT2_MAX_ENTRIES:
-    res.append(('SavedQuery_CLEANUP_WARNING', unicode(_('Saved Query has more than %d entries: %d, '
+    res.append(('SavedQuery_CLEANUP_WARNING', new_str(_('Saved Query has more than %d entries: %d, '
                 'please run "hue desktop_document_cleanup --cm-managed" to remove old entries' % (DOCUMENT2_MAX_ENTRIES, sq_count)))))
 
   job_count = Job.objects.count()
   if job_count > DOCUMENT2_MAX_ENTRIES:
-    res.append(('OOZIEJOB_CLEANUP_WARNING', unicode(_('Oozie Job has more than %d entries: %d, '
+    res.append(('OOZIEJOB_CLEANUP_WARNING', new_str(_('Oozie Job has more than %d entries: %d, '
                 'please run "hue desktop_document_cleanup --cm-managed" to remove old entries' % (DOCUMENT2_MAX_ENTRIES, job_count)))))
 
   if not get_secret_key():
-    res.append((SECRET_KEY, unicode(_("Secret key should be configured as a random string. All sessions will be lost on restart"))))
+    res.append((SECRET_KEY, new_str(_("Secret key should be configured as a random string. All sessions will be lost on restart"))))
 
   # Validate SSL setup
   if SSL_CERTIFICATE.get():
     res.extend(validate_path(SSL_CERTIFICATE, is_dir=False))
     if not SSL_PRIVATE_KEY.get():
-      res.append((SSL_PRIVATE_KEY, unicode(_("SSL private key file should be set to enable HTTPS."))))
+      res.append((SSL_PRIVATE_KEY, new_str(_("SSL private key file should be set to enable HTTPS."))))
     else:
       res.extend(validate_path(SSL_PRIVATE_KEY, is_dir=False))
 
   # Validate encoding
   if not i18n.validate_encoding(DEFAULT_SITE_ENCODING.get()):
-    res.append((DEFAULT_SITE_ENCODING, unicode(_("Encoding not supported."))))
+    res.append((DEFAULT_SITE_ENCODING, new_str(_("Encoding not supported."))))
 
   # Validate kerberos
   if KERBEROS.HUE_KEYTAB.get() is not None:
     res.extend(validate_path(KERBEROS.HUE_KEYTAB, is_dir=False))
     # Keytab should not be world or group accessible
     kt_stat = os.stat(KERBEROS.HUE_KEYTAB.get())
-    if stat.S_IMODE(kt_stat.st_mode) & 0077:
+    if stat.S_IMODE(kt_stat.st_mode) & 0o077:
       res.append((KERBEROS.HUE_KEYTAB,
                   force_unicode(_("Keytab should have 0600 permissions (has %o).") %
                   stat.S_IMODE(kt_stat.st_mode))))
@@ -2029,8 +2034,8 @@ def config_validator(user):
     from oozie.views.editor2 import _is_oozie_mail_enabled
 
     if not _is_oozie_mail_enabled(user):
-      res.append(('OOZIE_EMAIL_SERVER', unicode(_('Email notifications is disabled for Workflows and Jobs as SMTP server is localhost.'))))
-  except Exception, e:
+      res.append(('OOZIE_EMAIL_SERVER', new_str(_('Email notifications is disabled for Workflows and Jobs as SMTP server is localhost.'))))
+  except Exception as e:
     LOG.warn('Config check failed because Oozie app not installed %s' % e)
 
   from notebook.models import make_notebook
@@ -2041,13 +2046,13 @@ def config_validator(user):
   try:
     notebook_doc, save_as = _save_notebook(notebook.get_data(), user)
   except:
-    res.append(('DATABASE_CHARACTER_SET', unicode(_('Character set of <i>search</i> field in <i>desktop_document2</i> table is not UTF-8. <br>'
+    res.append(('DATABASE_CHARACTER_SET', new_str(_('Character set of <i>search</i> field in <i>desktop_document2</i> table is not UTF-8. <br>'
                                                     '<b>NOTE:</b> Configure the database for character set AL32UTF8 and national character set UTF8.'))))
   if notebook_doc:
     notebook_doc.delete()
 
   if 'use_new_editor' in USE_NEW_EDITOR.bind_to:
-    res.append(('[desktop] use_new_editor', unicode(_('This configuration flag has been deprecated.'))))
+    res.append(('[desktop] use_new_editor', new_str(_('This configuration flag has been deprecated.'))))
 
   return res
 

+ 7 - 7
desktop/core/src/desktop/configuration/api.py

@@ -34,7 +34,7 @@ from notebook.connectors.spark_shell import SparkConfiguration
 
 try:
   from oozie.models2 import WorkflowConfiguration as OozieWorkflowConfiguration
-except (ImportError, RuntimeError), e:
+except (ImportError, RuntimeError) as e:
   OozieWorkflowConfiguration = None
 
 
@@ -47,7 +47,7 @@ def api_error_handler(func):
 
     try:
       return func(*args, **kwargs)
-    except Exception, e:
+    except Exception as e:
       LOG.exception('Error running %s' % func)
       response['status'] = -1
       response['message'] = force_unicode(str(e))
@@ -111,7 +111,7 @@ def app_configuration_for_user(request):
 
     try:
       user = User.objects.get(id=int(user_id))
-    except User.DoesNotExist, e:
+    except User.DoesNotExist as e:
       raise PopupException(_('Could not find user with ID: %s') % user_id)
 
     config = _save_configuration(app, properties, is_default=False, user=user)
@@ -139,7 +139,7 @@ def delete_default_configuration(request):
     config_id = int(config_id)
     config = DefaultConfiguration.objects.get(id=config_id)
     config.delete()
-  except DefaultConfiguration.DoesNotExist, e:
+  except DefaultConfiguration.DoesNotExist as e:
     raise PopupException(_('Could not find configuration with ID: %d') % config_id)
 
   return JsonResponse({
@@ -204,7 +204,7 @@ def _update_default_and_group_configurations(configurations):
       # delete all previous default and group configurations
       DefaultConfiguration.objects.filter(Q(is_default=True) | Q(groups__isnull=False)).delete()
 
-      for app, configs in configurations.items():
+      for app, configs in list(configurations.items()):
         if 'default' in configs:
           properties = configs['default']
           if properties:
@@ -220,9 +220,9 @@ def _update_default_and_group_configurations(configurations):
               try:
                 groups = Group.objects.filter(id__in=group_ids)
                 _save_configuration(app, properties, is_default=False, groups=groups)
-              except Group.DoesNotExist, e:
+              except Group.DoesNotExist as e:
                 raise PopupException(_('Could not find one or more groups with IDs: %s') % ', '.join(group_ids))
-  except Exception, e:
+  except Exception as e:
     raise PopupException(_('Failed to update configurations: %s') % e)
 
   return _get_default_configurations()

+ 1 - 0
desktop/core/src/desktop/configuration/tests.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import json
 
 from nose.tools import assert_true, assert_false, assert_equal, assert_not_equal, assert_raises

+ 1 - 0
desktop/core/src/desktop/converter_tests.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 from datetime import datetime
 import json
 

+ 10 - 9
desktop/core/src/desktop/converters.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import json
 import logging
 import time
@@ -74,7 +75,7 @@ class DocumentConverter(object):
 
           # save() updates the last_modified to current time. Resetting it using update()
           Document2.objects.filter(id=doc.id).update(last_modified=doc_last_modified)
-      except Exception, e:
+      except Exception as e:
         LOG.exception("Failed to set is_trashed field with exception: %s" % e)
 
 
@@ -99,7 +100,7 @@ class DocumentConverter(object):
             )
 
             self.imported_doc_count += 1
-        except Exception, e:
+        except Exception as e:
           self.failed_doc_ids.append(doc.id)
           LOG.exception('Failed to import SavedQuery document id: %d' % doc.id)
     except ImportError:
@@ -135,10 +136,10 @@ class DocumentConverter(object):
 
               doc.add_tag(self.imported_tag)
               doc.save()
-        except Exception, e:
+        except Exception as e:
           self.failed_doc_ids.append(doc.id)
           LOG.exception('Failed to import history document id: %d' % doc.id)
-    except ImportError, e:
+    except ImportError as e:
       LOG.warn('Cannot convert history documents: beeswax app is not installed')
 
 
@@ -181,10 +182,10 @@ class DocumentConverter(object):
                 data=json.dumps(data)
               )
             self.imported_doc_count += 1
-        except Exception, e:
+        except Exception as e:
           self.failed_doc_ids.append(doc.id)
           LOG.exception('Failed to import Job Designer document id: %d' % doc.id)
-    except ImportError, e:
+    except ImportError as e:
       LOG.warn('Cannot convert Job Designer documents: oozie app is not installed')
 
 
@@ -210,10 +211,10 @@ class DocumentConverter(object):
             )
 
             self.imported_doc_count += 1
-        except Exception, e:
+        except Exception as e:
           self.failed_doc_ids.append(doc.id)
           LOG.exception('Failed to import Pig document id: %d' % doc.id)
-    except ImportError, e:
+    except ImportError as e:
       LOG.warn('Cannot convert Pig documents: pig app is not installed')
 
 
@@ -296,7 +297,7 @@ class DocumentConverter(object):
         document.add_tag(self.imported_tag)
         document.save()
         return document2
-    except Exception, e:
+    except Exception as e:
       # Just to be sure we delete Doc2 object incase of exception.
       # Possible when there are mixed InnoDB and MyISAM tables
       if document2 and Document2.objects.filter(id=document2.id).exists():

+ 19 - 1
desktop/core/src/desktop/lib/analytics/lib/hive.py

@@ -1,5 +1,23 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-class Hive():
+from builtins import object
+
+class Hive(object):
   NAME = 'Hive'
   TYPE = 'hive'
 

+ 19 - 1
desktop/core/src/desktop/lib/analytics/lib/impala.py

@@ -1,5 +1,23 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-class Impala():
+from builtins import object
+
+class Impala(object):
   NAME = 'Impala'
   TYPE = 'impala'
 

+ 29 - 23
desktop/core/src/desktop/lib/conf.py

@@ -63,6 +63,10 @@ variables.
 
 # The Config object unfortunately has a kwarg called "type", and everybody is
 # using it. So instead of breaking compatibility, we make a "pytype" alias.
+
+from __future__ import print_function
+from past.builtins import basestring
+from builtins import object
 pytype = type
 
 from django.utils.encoding import smart_str
@@ -84,6 +88,8 @@ try:
 except ImportError:
   from ordereddict import OrderedDict # Python 2.6
 
+if sys.version_info[0] > 2:
+  from past.builtins import long
 
 # Magical object for use as a "symbol"
 _ANONYMOUS = ("_ANONYMOUS")
@@ -304,14 +310,14 @@ class Config(object):
       req_kw = "required"
     else:
       req_kw = "optional"
-    print >>out, indent_str + "Key: %s (%s)" % (self.get_presentable_key(), req_kw)
+    print(indent_str + "Key: %s (%s)" % (self.get_presentable_key(), req_kw), file=out)
     if self.default_value:
-      print >>out, indent_str + "  Default: %s" % repr(self.default)
+      print(indent_str + "  Default: %s" % repr(self.default), file=out)
     elif self.dynamic_default:
-      print >>out, indent_str + "  Dynamic default: %s" % self.dynamic_default.__doc__.strip()
+      print(indent_str + "  Dynamic default: %s" % self.dynamic_default.__doc__.strip(), file=out)
 
-    print >>out, self.get_presentable_help_text(indent=indent)
-    print >>out
+    print(self.get_presentable_help_text(indent=indent), file=out)
+    print(file=out)
 
   def get_presentable_help_text(self, indent=0):
     indent_str = " " * indent
@@ -354,7 +360,7 @@ class BoundContainer(BoundConfig):
       return self.bind_to.setdefault(self.grab_key, {})
 
   def keys(self):
-    return self.get_data_dict().keys()
+    return list(self.get_data_dict().keys())
 
 class BoundContainerWithGetAttr(BoundContainer):
   """
@@ -397,7 +403,7 @@ class ConfigSection(Config):
     """
     super(ConfigSection, self).__init__(key, default={}, **kwargs)
     self.members = members or {}
-    for member in members.itervalues():
+    for member in members.values():
       assert member.key is not _ANONYMOUS
 
 
@@ -408,12 +414,12 @@ class ConfigSection(Config):
     @param new_members  A dictionary of {key=Config(...), key2=Config(...)}.
     @param overwrite  Whether to overwrite the current member on key conflict.
     """
-    for member in new_members.itervalues():
+    for member in new_members.values():
       assert member.key is not _ANONYMOUS
     if not overwrite:
       new_members = new_members.copy()
-      for k in self.members.iterkeys():
-        if new_members.has_key(k):
+      for k in self.members.keys():
+        if k in new_members:
           del new_members[k]
     self.members.update(new_members)
 
@@ -429,7 +435,7 @@ class ConfigSection(Config):
     are bound configuration parameters.
     """
     return dict([(key, self.get_member(raw, key, prefix))
-                 for key in self.members.iterkeys()])
+                 for key in self.members.keys()])
 
   def get_member(self, data, attr, prefix):
     if self.key is not _ANONYMOUS:
@@ -440,15 +446,15 @@ class ConfigSection(Config):
     if self.private:
       return
     if not skip_header:
-      print >>out, (" " * indent) + "[%s]" % self.get_presentable_key()
-      print >>out, self.get_presentable_help_text(indent=indent)
-      print >>out
+      print((" " * indent) + "[%s]" % self.get_presentable_key(), file=out)
+      print(self.get_presentable_help_text(indent=indent), file=out)
+      print(file=out)
       new_indent = indent + 2
     else:
       new_indent = indent
 
     # We sort the configuration for canonicalization.
-    for programmer_key, config in sorted(self.members.iteritems(), key=lambda x: x[1].key):
+    for programmer_key, config in sorted(iter(self.members.items()), key=lambda x: x[1].key):
       config.print_help(out=out, indent=new_indent)
 
 class UnspecifiedConfigSection(Config):
@@ -481,7 +487,7 @@ class UnspecifiedConfigSection(Config):
     The keys are the keys specified by the user in the config file.
     """
     return OrderedDict([(key, self.get_member(raw, key, prefix))
-                 for key in raw.iterkeys()])
+                 for key in raw.keys()])
 
   def get_member(self, data, attr, prefix=''):
     tail = self.key + '.' + attr
@@ -492,10 +498,10 @@ class UnspecifiedConfigSection(Config):
   def print_help(self, out=sys.stdout, indent=0):
     indent_str = " " * indent
 
-    print >>out, indent_str + "[%s]" % self.get_presentable_key()
-    print >>out, self.get_presentable_help_text(indent=indent)
-    print >>out
-    print >>out, indent_str + "  Consists of some number of sections like:"
+    print(indent_str + "[%s]" % self.get_presentable_key(), file=out)
+    print(self.get_presentable_help_text(indent=indent), file=out)
+    print(file=out)
+    print(indent_str + "  Consists of some number of sections like:", file=out)
     self.each.print_help(out=out, indent=indent+2)
 
 def _configs_from_dir(conf_dir):
@@ -509,7 +515,7 @@ def _configs_from_dir(conf_dir):
     LOG.debug("Loading configuration from: %s" % filename)
     try:
       conf = ConfigObj(os.path.join(conf_dir, filename))
-    except ConfigObjError, ex:
+    except ConfigObjError as ex:
       LOG.error("Error in configuration file '%s': %s" % (os.path.join(conf_dir, filename), ex))
       raise
     conf['DEFAULT'] = dict(desktop_root=get_desktop_root(), build_dir=get_build_dir())
@@ -539,7 +545,7 @@ def _bind_module_members(module, data, section):
   Returns the dict of unbound configs.
   """
   members = {}
-  for key, val in module.__dict__.iteritems():
+  for key, val in module.__dict__.items():
     if not isinstance(val, Config):
       continue
 
@@ -658,7 +664,7 @@ def list_of_compiled_res(skip_empty=False):
   def fn(list_of_strings):
     if isinstance(list_of_strings, basestring):
       list_of_strings = list_of_strings.split(',')
-    list_of_strings = filter(lambda string: string if skip_empty else True, list_of_strings)
+    list_of_strings = [string if skip_empty else True for string in list_of_strings]
     return list(re.compile(x) for x in list_of_strings)
   return fn
 

+ 12 - 4
desktop/core/src/desktop/lib/conf_test.py

@@ -15,14 +15,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
 import configobj
-from cStringIO import StringIO
 import logging
 import re
+import sys
 
 from desktop.lib.conf import *
 from nose.tools import assert_true, assert_false, assert_equals, assert_raises
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from cStringIO import StringIO as string_io
+
 def my_dynamic_default():
   """
   Calculates a sum
@@ -83,8 +91,8 @@ class TestConfig(object):
                          PORT = Config("port", help="Thrift port for the NN",
                                        type=int, default=10090))))))
     cls.conf = cls.conf.bind(
-      load_confs([configobj.ConfigObj(infile=StringIO(cls.CONF_ONE)),
-                  configobj.ConfigObj(infile=StringIO(cls.CONF_TWO))]),
+      load_confs([configobj.ConfigObj(infile=string_io(cls.CONF_ONE)),
+                  configobj.ConfigObj(infile=string_io(cls.CONF_TWO))]),
       prefix='')
 
   def test_type_safety(self):
@@ -180,7 +188,7 @@ class TestConfig(object):
     assert_raises(Exception, coerce_bool, tuple("foo"))
 
   def test_print_help(self):
-    out = StringIO()
+    out = string_io()
     self.conf.print_help(out=out, skip_header=True)
     out = out.getvalue().strip()
     assert_false("dontseeme" in out)

+ 3 - 2
desktop/core/src/desktop/lib/config_spec_dump.py

@@ -14,11 +14,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from builtins import object
 import desktop.appmanager
 
 from desktop.lib.conf import BoundContainer, UnspecifiedConfigSection, is_anonymous
 
-class ConfigSpec():
+class ConfigSpec(object):
   def __init__(self, configspec):
     self.indent = 0
     self.level = 0
@@ -42,7 +43,7 @@ class ConfigSpec():
       self.indent += 2
       self.level += 1
       sections = []
-      for v in config_obj.get().values():
+      for v in list(config_obj.get().values()):
         if isinstance(v, BoundContainer):
           sections.append(v)
         else:

+ 3 - 3
desktop/core/src/desktop/lib/connectors/api.py

@@ -174,7 +174,7 @@ def update_connector(request):
 def _get_connector_by_type(dialect):
   global CONNECTOR_TYPES
 
-  instance = filter(lambda connector: connector['dialect'] == dialect, CONNECTOR_TYPES)
+  instance = [connector for connector in CONNECTOR_TYPES if connector['dialect'] == dialect]
 
   if instance:
     return instance[0]
@@ -188,7 +188,7 @@ def delete_connector(request):
   connector = json.loads(request.POST.get('connector'), '{}')
 
   size_before = len(CONNECTOR_INSTANCES)
-  CONNECTOR_INSTANCES = filter(lambda _connector: _connector['name'] != connector['name'], CONNECTOR_INSTANCES)
+  CONNECTOR_INSTANCES = [_connector for _connector in CONNECTOR_INSTANCES if _connector['name'] != connector['name']]
   size_after = len(CONNECTOR_INSTANCES)
 
   if size_before == size_after + 1:
@@ -230,7 +230,7 @@ def _get_installed_connectors():
 def _get_connector_by_id(id):
   global CONNECTOR_INSTANCES
 
-  instance = filter(lambda connector: connector['id'] == id, CONNECTOR_INSTANCES)
+  instance = [connector for connector in CONNECTOR_INSTANCES if connector['id'] == id]
 
   if instance:
     return instance[0]

+ 19 - 1
desktop/core/src/desktop/lib/connectors/lib/hive.py

@@ -1,5 +1,23 @@
+#!/usr/bin/python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-class Hive():
+from builtins import object
+
+class Hive(object):
   NAME = 'Hive'
   TYPE = 'hive'
 

+ 19 - 1
desktop/core/src/desktop/lib/connectors/lib/impala.py

@@ -1,5 +1,23 @@
+#!/usr/bin/python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-class Impala():
+from builtins import object
+
+class Impala(object):
   NAME = 'Impala'
   TYPE = 'impala'
 

+ 20 - 10
desktop/core/src/desktop/lib/django_forms.py

@@ -17,9 +17,14 @@
 #
 # Extra form fields and widgets.
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import filter
+from builtins import range
+from builtins import object
 import logging
 import json
-import urllib
+import sys
 
 from django.forms import Widget, Field
 from django import forms
@@ -33,6 +38,11 @@ from django.utils.encoding import python_2_unicode_compatible, force_unicode
 import desktop.lib.i18n
 from desktop.lib.i18n import smart_str
 
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+  from urllib.parse import quote_plus as urllib_quote_plus
+else:
+  from urllib import quote_plus as urllib_quote_plus
 
 LOG = logging.getLogger(__name__)
 
@@ -113,7 +123,7 @@ class MultipleInputWidget(Widget):
     # Sometimes this is a QueryDict, and sometimes ar regular dict,
     # so we adapt:
     non_empty = lambda x: len(x) != 0
-    return filter(non_empty, data.getlist(name))
+    return list(filter(non_empty, data.getlist(name)))
 
 class MultipleInputField(Field):
   widget = MultipleInputWidget
@@ -188,7 +198,7 @@ class KeyValueWidget(Textarea):
   def render(self, name, value, attrs=None):
     # If we have a dictionary, render back into a string.
     if isinstance(value, dict):
-      value = " ".join("=".join([k, v]) for k, v in value.iteritems())
+      value = " ".join("=".join([k, v]) for k, v in value.items())
     return super(KeyValueWidget, self).render(name, value, attrs)
 
 class KeyValueField(CharField):
@@ -273,12 +283,12 @@ class MultiForm(object):
   def get_subforms(self):
     """get_subforms() -> An iterator over (name, subform)"""
     assert self._is_bound
-    return self._forms.iteritems()
+    return iter(self._forms.items())
 
   def has_subform_data(self, subform_name, data):
     """Test if data contains any information bound for the subform"""
     prefix = self.add_prefix(subform_name)
-    return len([ k.startswith(prefix) for k in data.keys() ]) != 0
+    return len([ k.startswith(prefix) for k in list(data.keys()) ]) != 0
 
   def add_subform(self, name, form_cls, data=None):
     """Dynamically extend this MultiForm to include a new subform"""
@@ -288,13 +298,13 @@ class MultiForm(object):
   def remove_subform(self, name):
     """Dynamically remove a subform. Raises KeyError."""
     del self._form_types[name]
-    if self._forms.has_key(name):
+    if name in self._forms:
       del self._forms[name]
 
   def bind(self, data=None, instances=None):
     self._is_bound = True
     self._forms = {}
-    for key, form_cls in self._form_types.iteritems():
+    for key, form_cls in self._form_types.items():
       instance = instances is not None and instances.get(key) or None
       self._bind_one(key, form_cls, data, instance=instance)
 
@@ -318,7 +328,7 @@ class MultiForm(object):
     r = True
     # Explicitly iterate through all of them; we don't want
     # to abort early, since we want each form's is_valid to be run.
-    for f in self._forms.values():
+    for f in list(self._forms.values()):
       if not f.is_valid():
         LOG.error(smart_str(f.errors))
         r = False
@@ -472,7 +482,7 @@ class BaseSimpleFormSet(StrAndUnicode):
       self._errors.append(f.errors)
     try:
       self.clean()
-    except ValidationError, e:
+    except ValidationError as e:
       self._non_form_errors = e.messages
 
   @property
@@ -541,4 +551,4 @@ class DependencyAwareForm(forms.Form):
     return [ data(*x) for x in self.dependencies ]
 
   def render_dep_metadata(self):
-    return urllib.quote_plus(json.dumps(self._calculate_data(), separators=(',', ':')))
+    return urllib_quote_plus(json.dumps(self._calculate_data(), separators=(',', ':')))

+ 2 - 2
desktop/core/src/desktop/lib/django_mako.py

@@ -112,7 +112,7 @@ def render_to_string_normal(template_name, django_context):
     data_dict = django_context
 
   template = lookup.get_template(template_name)
-  data_dict = dict(map(lambda k: (str(k), data_dict.get(k)), data_dict.keys()))
+  data_dict = dict([(str(k), data_dict.get(k)) for k in list(data_dict.keys())])
   result = template.render(**data_dict)
   return i18n.smart_unicode(result)
 
@@ -138,7 +138,7 @@ def csrf_token(request):
   """
   Returns the rendered common footer
   """
-  csrf_token = unicode(csrf(request)["csrf_token"])
+  csrf_token = str(csrf(request)["csrf_token"])
   return str.format("<input type='hidden' name='csrfmiddlewaretoken' value='{0}' />", csrf_token)
 
 def static(path):

+ 5 - 4
desktop/core/src/desktop/lib/django_util.py

@@ -17,6 +17,7 @@
 #
 # Utilities for django operations.
 
+from builtins import object
 import logging
 import re
 import json
@@ -54,7 +55,7 @@ GROUPNAME_RE_RULE = ".{,80}"
 
 
 # For backward compatibility for upgrades to Hue 2.2
-class PopupException: pass
+class PopupException(object): pass
 
 
 class Encoder(json.JSONEncoder):
@@ -114,7 +115,7 @@ def copy_query_dict(query_dict, attr_list):
   """
   res = QueryDict('', mutable=True)
   for attr in attr_list:
-    if query_dict.has_key(attr):
+    if attr in query_dict:
       res[attr] = query_dict.get(attr)
   return res
 
@@ -313,7 +314,7 @@ def update_if_dirty(model_instance, **kwargs):
   saves only if there's been a change.
   """
   dirty = False
-  for key, value in kwargs.items():
+  for key, value in list(kwargs.items()):
     if getattr(model_instance, key) != value:
       setattr(model_instance, key, value)
       dirty = True
@@ -345,7 +346,7 @@ class TruncatingModel(models.Model):
   Abstract class which truncates Text and Char fields to their configured
   maximum lengths, to avoid database field overflow errors.
   """
-  class Meta:
+  class Meta(object):
     abstract = True
 
   def __setattr__(self, name, value):

+ 7 - 2
desktop/core/src/desktop/lib/django_util_test.py

@@ -16,7 +16,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import datetime
+import sys
 
 from nose.tools import assert_true, assert_equal, assert_not_equal, assert_raises
 from django.http import HttpResponse, HttpResponseRedirect
@@ -28,8 +30,11 @@ configure_django_for_test()
 from desktop.lib import django_util, exceptions
 from django.db import models
 
+if sys.version_info[0] > 2:
+  unichr = chr
+
 class TestModel(models.Model):
-  class Meta:
+  class Meta(object):
     app_label = "TEST_APP"
 
   my_int = models.IntegerField()
@@ -136,7 +141,7 @@ class TestDjangoUtil(object):
     the_file = "foobar"
     try:
       raise exceptions.MessageException(msg, the_file)
-    except Exception, e:
+    except Exception as e:
       assert_equal(msg, e.message)
       assert_equal(the_file, e.data['filename'])
       assert_true(msg in str(e))

+ 1 - 1
desktop/core/src/desktop/lib/djangojson.py

@@ -42,7 +42,7 @@ class JsonFormField(fields.CharField):
     value = super(JsonFormField, self).clean(value)
     try:
       json.loads(value)
-    except ValueError, e:
+    except ValueError as e:
       raise util.ValidationError(e)
     return value
   

+ 3 - 1
desktop/core/src/desktop/lib/djangothrift.py

@@ -39,9 +39,11 @@
 # been through this dirtying, and that's bad.
 # So, don't use JSON serialization, and use XML serialization instead.
 
+from __future__ import absolute_import
+from past.builtins import basestring
 import json
 
-import thrift_util
+from . import thrift_util
 
 from django.db import models
 

+ 5 - 3
desktop/core/src/desktop/lib/djangothrift_test.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 # Tests for ThriftField.
 # Some parts based on http://www.djangosnippets.org/snippets/1044/
 # Licensed to Cloudera, Inc. under one
@@ -16,23 +17,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import os
 import sys
 sys.path.insert(1, os.path.join(os.path.dirname(__file__), "gen-py"))
 from djangothrift_test_gen.ttypes import TestStruct
 import unittest
 
-from django_test_util import configure_django_for_test, create_tables
+from desktop.lib.django_test_util import configure_django_for_test, create_tables
 
 configure_django_for_test()
 
 from django.db import models
-from djangothrift import ThriftField
+from desktop.lib.djangothrift import ThriftField
 
 from desktop.lib import django_util
 
 class ThriftTestModel(models.Model):
-  class Meta:
+  class Meta(object):
     app_label = "TEST_THRIFT_APP"
 
   my_int = models.IntegerField()

+ 11 - 3
desktop/core/src/desktop/lib/export_csvxls.py

@@ -18,13 +18,17 @@
 """
 Common library to export either CSV or XLS.
 """
+from future import standard_library
+standard_library.install_aliases()
+from builtins import next
+from builtins import object
 import gc
 import logging
 import numbers
 import openpyxl
 import re
 import six
-import StringIO
+import sys
 import tablib
 
 from django.http import StreamingHttpResponse, HttpResponse
@@ -32,6 +36,10 @@ from django.utils.encoding import smart_str
 from django.utils.http import urlquote
 from desktop.lib import i18n
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from StringIO import StringIO as string_io
 
 LOG = logging.getLogger(__name__)
 
@@ -82,13 +90,13 @@ def dataset(headers, data, encoding=None):
   return dataset
 
 
-class XlsWrapper():
+class XlsWrapper(object):
   def __init__(self, xls):
     self.xls = xls
 
 
 def xls_dataset(workbook):
-  output = StringIO.StringIO()
+  output = string_io()
   workbook.save(output)
   output.seek(0)
   return XlsWrapper(output.read())

+ 11 - 3
desktop/core/src/desktop/lib/fs/__init__.py

@@ -16,14 +16,22 @@
 
 from __future__ import absolute_import
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import filter
 import posixpath
-import urlparse
+import sys
+import urllib.parse
 
 from desktop.lib.fs.proxyfs import ProxyFS
 
+if sys.version_info[0] > 2:
+  from urllib.parse import urlparse as lib_urlparse
+else:
+  from urlparse import urlparse as lib_urlparse
 
 def splitpath(path):
-  split = urlparse.urlparse(path)
+  split = lib_urlparse(path)
   if split.scheme and split.netloc:
     parts = [split.scheme + '://', split.netloc] + split.path.split('/')
   elif split.scheme and split.path:
@@ -33,7 +41,7 @@ def splitpath(path):
   else:
     parts = ['/'] + posixpath.normpath(path).split('/')
   # Filter empty parts out
-  return filter(len, parts)
+  return list(filter(len, parts))
 
 
 

+ 13 - 5
desktop/core/src/desktop/lib/fs/proxyfs.py

@@ -16,14 +16,22 @@
 
 from __future__ import absolute_import
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
 import logging
+import sys
 
-from urlparse import urlparse
 from django.contrib.auth.models import User
 
 from desktop.auth.backend import is_admin
 from desktop.conf import DEFAULT_USER
 
+if sys.version_info[0] > 2:
+  from urllib.parse import urlparse as lib_urlparse
+else:
+  from urlparse import urlparse as lib_urlparse
+
 LOG = logging.getLogger(__name__)
 
 DEFAULT_USER = DEFAULT_USER.get()
@@ -33,7 +41,7 @@ class ProxyFS(object):
   def __init__(self, filesystems_dict, default_scheme, name='default'):
     if default_scheme not in filesystems_dict:
       raise ValueError(
-        'Default scheme "%s" is not a member of provided schemes: %s' % (default_scheme, filesystems_dict.keys()))
+        'Default scheme "%s" is not a member of provided schemes: %s' % (default_scheme, list(filesystems_dict.keys())))
 
     self._name = name
     self._fs_dict = filesystems_dict
@@ -53,7 +61,7 @@ class ProxyFS(object):
   def _get_scheme(self, path):
     scheme = None
     if path:
-      split = urlparse(path)
+      split = lib_urlparse(path)
       scheme = split.scheme if split.scheme else None
     ret_scheme = scheme or self._default_scheme
     if not ret_scheme:
@@ -85,7 +93,7 @@ class ProxyFS(object):
       else:
         raise IOError("Missing permissions for %s on %s" % (self.getuser(), path))
     except KeyError:
-      raise IOError('Unknown scheme %s, available schemes: %s' % (scheme, self._fs_dict.keys()))
+      raise IOError('Unknown scheme %s, available schemes: %s' % (scheme, list(self._fs_dict.keys())))
 
   def _get_fs_pair(self, src, dst):
     """
@@ -94,7 +102,7 @@ class ProxyFS(object):
     """
 
     src_fs = self._get_fs(src)
-    dst_scheme = urlparse(dst).scheme
+    dst_scheme = lib_urlparse(dst).scheme
     if not dst_scheme:
       return src_fs, src_fs
     return src_fs, self._get_fs(dst)

+ 2 - 1
desktop/core/src/desktop/lib/fs/proxyfs_test.py

@@ -16,6 +16,7 @@
 
 from __future__ import absolute_import
 
+from builtins import object
 from mock import MagicMock, patch
 from nose.plugins.attrib import attr
 from nose.tools import assert_raises, assert_false, eq_
@@ -102,7 +103,7 @@ def test_constructor_given_invalid_arguments():
   assert_raises(ValueError, ProxyFS, {'s3a': {}}, 'hdfs')
 
 
-class MockFs():
+class MockFs(object):
   def __init__(self, filebrowser_action=None):
     self.user = None
     self._filebrowser_action = filebrowser_action

+ 5 - 3
desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/TestService.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 #
 # Autogenerated by Thrift Compiler (0.9.0)
 #
@@ -6,8 +7,9 @@
 #  options string: py:new_style
 #
 
+from builtins import object
 from thrift.Thrift import TType, TMessageType, TException, TApplicationException
-from ttypes import *
+from djangothrift_test_gen.ttypes import *
 from thrift.Thrift import TProcessor
 from thrift.transport import TTransport
 from thrift.protocol import TBinaryProtocol, TProtocol
@@ -150,7 +152,7 @@ class ping_args(object):
 
   def __repr__(self):
     L = ['%s=%r' % (key, value)
-      for key, value in self.__dict__.iteritems()]
+      for key, value in self.__dict__.items()]
     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
 
   def __eq__(self, other):
@@ -209,7 +211,7 @@ class ping_result(object):
 
   def __repr__(self):
     L = ['%s=%r' % (key, value)
-      for key, value in self.__dict__.iteritems()]
+      for key, value in self.__dict__.items()]
     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
 
   def __eq__(self, other):

+ 2 - 1
desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/constants.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 #
 # Autogenerated by Thrift Compiler (0.9.0)
 #
@@ -7,5 +8,5 @@
 #
 
 from thrift.Thrift import TType, TMessageType, TException, TApplicationException
-from ttypes import *
+from desktop.lib.djangothrift_test_gen.ttypes import *
 

+ 10 - 8
desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/ttypes.py

@@ -6,6 +6,8 @@
 #  options string: py:new_style
 #
 
+from builtins import range
+from builtins import object
 from thrift.Thrift import TType, TMessageType, TException, TApplicationException
 
 from thrift.transport import TTransport
@@ -97,7 +99,7 @@ class TestStruct(object):
 
   def __repr__(self):
     L = ['%s=%r' % (key, value)
-      for key, value in self.__dict__.iteritems()]
+      for key, value in self.__dict__.items()]
     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
 
   def __eq__(self, other):
@@ -170,7 +172,7 @@ class TestNesting(object):
 
   def __repr__(self):
     L = ['%s=%r' % (key, value)
-      for key, value in self.__dict__.iteritems()]
+      for key, value in self.__dict__.items()]
     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
 
   def __eq__(self, other):
@@ -299,7 +301,7 @@ class TestManyTypes(object):
         if ftype == TType.SET:
           self.a_set = set()
           (_etype3, _size0) = iprot.readSetBegin()
-          for _i4 in xrange(_size0):
+          for _i4 in range(_size0):
             _elem5 = iprot.readI32();
             self.a_set.add(_elem5)
           iprot.readSetEnd()
@@ -309,7 +311,7 @@ class TestManyTypes(object):
         if ftype == TType.LIST:
           self.a_list = []
           (_etype9, _size6) = iprot.readListBegin()
-          for _i10 in xrange(_size6):
+          for _i10 in range(_size6):
             _elem11 = TestStruct()
             _elem11.read(iprot)
             self.a_list.append(_elem11)
@@ -320,7 +322,7 @@ class TestManyTypes(object):
         if ftype == TType.MAP:
           self.a_map = {}
           (_ktype13, _vtype14, _size12 ) = iprot.readMapBegin() 
-          for _i16 in xrange(_size12):
+          for _i16 in range(_size12):
             _key17 = iprot.readI32();
             _val18 = TestStruct()
             _val18.read(iprot)
@@ -337,7 +339,7 @@ class TestManyTypes(object):
         if ftype == TType.LIST:
           self.a_string_list = []
           (_etype22, _size19) = iprot.readListBegin()
-          for _i23 in xrange(_size19):
+          for _i23 in range(_size19):
             _elem24 = iprot.readString();
             self.a_string_list.append(_elem24)
           iprot.readListEnd()
@@ -410,7 +412,7 @@ class TestManyTypes(object):
     if self.a_map is not None:
       oprot.writeFieldBegin('a_map', TType.MAP, 13)
       oprot.writeMapBegin(TType.I32, TType.STRUCT, len(self.a_map))
-      for kiter27,viter28 in self.a_map.items():
+      for kiter27,viter28 in list(self.a_map.items()):
         oprot.writeI32(kiter27)
         viter28.write(oprot)
       oprot.writeMapEnd()
@@ -435,7 +437,7 @@ class TestManyTypes(object):
 
   def __repr__(self):
     L = ['%s=%r' % (key, value)
-      for key, value in self.__dict__.iteritems()]
+      for key, value in self.__dict__.items()]
     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
 
   def __eq__(self, other):

+ 1 - 0
desktop/core/src/desktop/lib/idbroker/client.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 from __future__ import absolute_import
 
+from builtins import object
 import logging
 
 from desktop.lib.exceptions_renderable import PopupException

+ 1 - 1
desktop/core/src/desktop/lib/metrics/file_reporter.py

@@ -38,7 +38,7 @@ class FileReporter(Reporter):
     if not os.path.exists(dirname):
       try:
         os.makedirs(dirname)
-      except OSError, e:
+      except OSError as e:
         LOG.error('failed to make the directory %s: %s' % (dirname, e))
       return
 

+ 2 - 1
desktop/core/src/desktop/lib/metrics/registry.py

@@ -18,6 +18,7 @@
 All Hue metrics should be defined in the APP/metrics.py file so they are discoverable.
 """
 
+from builtins import object
 import functools
 import pyformance
 
@@ -107,7 +108,7 @@ class MetricsRegistry(object):
     metrics = self._registry.dump_metrics()
 
     # Filter out min and max if there have been no samples.
-    for metric in metrics.itervalues():
+    for metric in metrics.values():
       if metric.get('count') == 0:
         if 'min' in metric:
           metric['min'] = 0.0

+ 4 - 3
desktop/core/src/desktop/lib/parameterization.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from past.builtins import basestring
 import logging
 
 from string import Template
@@ -40,7 +41,7 @@ def recursive_walk(function, data):
 
   # Recurse into dicts:
   if isinstance(data, dict):
-    return dict( (key, recursive_walk(function, val)) for key, val in data.iteritems() )
+    return dict( (key, recursive_walk(function, val)) for key, val in data.items() )
 
   return function(data)
 
@@ -82,7 +83,7 @@ def substitute_variables(input_data, substitutions):
 def find_parameters(obj, fields=None):
   """Find parameters in the given fields"""
   if fields is None:
-    fields = [ k for k in obj.__dict__.keys() if not k.startswith('_') ]
+    fields = [ k for k in list(obj.__dict__.keys()) if not k.startswith('_') ]
 
   params = [ ]
   for field in fields:
@@ -98,7 +99,7 @@ def find_parameters(obj, fields=None):
 def bind_parameters(obj, substitutions, fields=None):
   """Bind the parameters to the given fields, changing their values."""
   if fields is None:
-    fields = [ k for k in obj.__dict__.keys() if not k.startswith('_') ]
+    fields = [ k for k in list(obj.__dict__.keys()) if not k.startswith('_') ]
 
   for field in fields:
     data = getattr(obj, field)

+ 2 - 2
desktop/core/src/desktop/lib/paths.py

@@ -91,8 +91,8 @@ def get_run_root(*append):
 
 def get_hadoop_conf_dir_default_config():
   from hadoop.conf import HDFS_CLUSTERS, get_hadoop_conf_dir_default
-  if HDFS_CLUSTERS.keys():
-    yarn_site_path = HDFS_CLUSTERS[HDFS_CLUSTERS.keys()[0]].HADOOP_CONF_DIR.get()
+  if list(HDFS_CLUSTERS.keys()):
+    yarn_site_path = HDFS_CLUSTERS[list(HDFS_CLUSTERS.keys())[0]].HADOOP_CONF_DIR.get()
   else:
     yarn_site_path = get_hadoop_conf_dir_default()
   return yarn_site_path

+ 2 - 0
desktop/core/src/desktop/lib/python_util.py

@@ -17,6 +17,8 @@
 # 
 # Extra python utils
 
+from past.builtins import basestring
+from builtins import object
 import select
 import socket
 from django.utils.translation import ugettext as _

+ 1 - 0
desktop/core/src/desktop/lib/python_util_test.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import datetime
 
 from nose.tools import assert_true, assert_equal, assert_not_equal

+ 13 - 6
desktop/core/src/desktop/lib/rest/http_client.py

@@ -14,13 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
 import logging
 import posixpath
 import requests
 import threading
-import urllib
-
-from urlparse import urlparse
+import sys
 
 from django.utils.encoding import iri_to_uri, smart_str
 from django.utils.http import urlencode
@@ -32,6 +33,12 @@ from requests.auth import AuthBase ,HTTPBasicAuth, HTTPDigestAuth
 from requests_kerberos import HTTPKerberosAuth, REQUIRED, OPTIONAL, DISABLED
 from urllib3.contrib import pyopenssl
 
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+  from urllib.parse import quote as urllib_quote, urlparse as lib_urlparse
+else:
+  from urllib import quote as urllib_quote
+  from urlparse import urlparse as lib_urlparse
 
 pyopenssl.DEFAULT_SSL_CIPHER_LIST = conf.SSL_CIPHER_LIST.get()
 
@@ -112,7 +119,7 @@ class HttpClient(object):
     self._cookies = None
 
   def _extract_netloc(self, base_url):
-    parsed_uri = urlparse(base_url)
+    parsed_uri = lib_urlparse(base_url)
     short_url = '%(scheme)s://%(netloc)s' % {'scheme': parsed_uri.scheme, 'netloc': parsed_uri.netloc}
     return short_url
 
@@ -184,7 +191,7 @@ class HttpClient(object):
     """
     # Prepare URL and params
     if urlencode:
-      path = urllib.quote(smart_str(path))
+      path = urllib_quote(smart_str(path))
     url = self._make_url(path, params)
     if http_method in ("GET", "DELETE"):
       if data is not None:
@@ -217,7 +224,7 @@ class HttpClient(object):
             exceptions.HTTPError,
             exceptions.RequestException,
             exceptions.URLRequired,
-            exceptions.TooManyRedirects), ex:
+            exceptions.TooManyRedirects) as ex:
       raise self._exc_class(ex)
 
   def _make_url(self, path, params):

+ 3 - 2
desktop/core/src/desktop/lib/rest/http_client_test.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 from requests import Response
 from requests.exceptions import HTTPError
 from desktop.conf import REST_CONN_TIMEOUT
@@ -38,7 +39,7 @@ def test_http_error_rest_exception():
   assert_equal(headers, exception._headers)
 
 
-class MockedSession():
+class MockedSession(object):
 
   def __init__(self, cookies=None):
     self.cookies = cookies or {}
@@ -47,7 +48,7 @@ class MockedSession():
     return MockedResponse()
 
 
-class MockedResponse():
+class MockedResponse(object):
   def __init__(self, status_code=200, cookies=None):
     self.status_code = status_code
     self.cookies = cookies

+ 6 - 3
desktop/core/src/desktop/lib/rest/resource.py

@@ -1,3 +1,4 @@
+from __future__ import division
 # Licensed to Cloudera, Inc. under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from past.utils import old_div
+from builtins import object
 import logging
 import posixpath
 import time
@@ -62,7 +65,7 @@ class Resource(object):
           'application/json' in resp.headers.get('content-type'):
       try:
         return resp.json()
-      except Exception, ex:
+      except Exception as ex:
         self._client.logger.exception('JSON decode error: %s' % resp.content)
         raise ex
     else:
@@ -200,9 +203,9 @@ class Resource(object):
 
 # Same in thrift_util.py for not losing the trace class
 def log_if_slow_call(duration, message, logger):
-  if duration >= WARN_LEVEL_CALL_DURATION_MS / 1000:
+  if duration >= old_div(WARN_LEVEL_CALL_DURATION_MS, 1000):
     logger.warn('SLOW: %.2f - %s' % (duration, message))
-  elif duration >= INFO_LEVEL_CALL_DURATION_MS / 1000:
+  elif duration >= old_div(INFO_LEVEL_CALL_DURATION_MS, 1000):
     logger.info('SLOW: %.2f - %s' % (duration, message))
   else:
     logger.debug(message)

+ 1 - 1
desktop/core/src/desktop/lib/scheduler/api.py

@@ -59,7 +59,7 @@ def submit_schedule(request, doc_id):
       jsonify = request.POST.get('format') == 'json'
       try:
         job_id = get_api(request, interface).submit_schedule(request, coordinator, mapping)
-      except Exception, e:
+      except Exception as e:
         message = force_unicode(str(e))
         return JsonResponse({'status': -1, 'message': message}, safe=False)
       if jsonify:

+ 2 - 1
desktop/core/src/desktop/lib/scheduler/lib/api.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 
 
+from builtins import object
 def get_api(request, interface):
   if interface == 'beat':
     from desktop.lib.scheduler.lib.beat import CeleryBeatApi
@@ -27,7 +28,7 @@ def get_api(request, interface):
     raise PopupException(_('Scheduler connector interface not recognized: %s') % interface)
 
 
-class Api():
+class Api(object):
 
   def __init__(self, interface, user):
     self.interface = interface

+ 1 - 0
desktop/core/src/desktop/lib/scheduler/lib/oozie.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 
 from desktop.lib.scheduler.lib.api import Api
 

+ 13 - 5
desktop/core/src/desktop/lib/tasks/compress_files/compress_utils.py

@@ -15,8 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from future import standard_library
+standard_library.install_aliases()
 import json
-import urllib
+import sys
 
 from django.urls import reverse
 from django.utils.translation import ugettext as _
@@ -26,18 +28,24 @@ from desktop.lib.paths import get_desktop_root, SAFE_CHARACTERS_URI_COMPONENTS,
 
 from notebook.connectors.base import Notebook
 
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+  from urllib.parse import quote as urllib_quote
+else:
+  from urllib import quote as urllib_quote
+
 def compress_files_in_hdfs(request, file_names, upload_path, archive_name):
 
   _upload_compress_files_script_to_hdfs(request.fs)
 
-  files = [{"value": upload_path + '/' + urllib.quote(file_name.encode('utf-8'), SAFE_CHARACTERS_URI)} for file_name in file_names]
+  files = [{"value": upload_path + '/' + urllib_quote(file_name.encode('utf-8'), SAFE_CHARACTERS_URI)} for file_name in file_names]
   files.append({'value': '/user/' + DEFAULT_USER.get() + '/common/compress_files_in_hdfs.sh'})
   start_time = json.loads(request.POST.get('start_time', '-1'))
 
   shell_notebook = Notebook(
     name=_('HDFS Compression to %(upload_path)s/hue_compressed.zip') % {'upload_path': upload_path},
     isManaged=True,
-    onSuccessUrl='/filebrowser/view=' + urllib.quote(upload_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+    onSuccessUrl='/filebrowser/view=' + urllib_quote(upload_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
   )
    
   shell_notebook.add_shell_snippet(
@@ -54,9 +62,9 @@ def compress_files_in_hdfs(request, file_names, upload_path, archive_name):
 def _upload_compress_files_script_to_hdfs(fs):
   if not fs.exists('/user/' + DEFAULT_USER.get() + '/common/'):
     fs.do_as_user(DEFAULT_USER.get(), fs.mkdir, '/user/' + DEFAULT_USER.get() + '/common/')
-    fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0755)
+    fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0o755)
 
   if not fs.do_as_user(DEFAULT_USER.get(), fs.exists, '/user/' + DEFAULT_USER.get() + '/common/compress_files_in_hdfs.sh'):
     fs.do_as_user(DEFAULT_USER.get(), fs.copyFromLocal, get_desktop_root() + '/core/src/desktop/lib/tasks/compress_files/compress_in_hdfs.sh',
                           '/user/' + DEFAULT_USER.get() + '/common/compress_files_in_hdfs.sh')
-    fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0755)
+    fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0o755)

+ 7 - 5
desktop/core/src/desktop/lib/tasks/extract_archive/extract_utils.py

@@ -15,8 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from future import standard_library
+standard_library.install_aliases()
 import json
-import urllib
+import urllib.request, urllib.parse, urllib.error
 
 from django.urls import reverse
 from django.utils.translation import ugettext as _
@@ -36,14 +38,14 @@ def extract_archive_in_hdfs(request, upload_path, file_name):
   shell_notebook = Notebook(
       name=_('HDFS Extraction of %(upload_path)s/%(file_name)s') % {'upload_path': upload_path, 'file_name': file_name},
       isManaged=True,
-      onSuccessUrl='/filebrowser/view=' + urllib.quote(output_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+      onSuccessUrl='/filebrowser/view=' + urllib.parse.quote(output_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
   )
 
   shell_notebook.add_shell_snippet(
       shell_command='extract_archive_in_hdfs.sh',
       arguments=[{'value': '-u=' + upload_path}, {'value': '-f=' + file_name}, {'value': '-o=' + output_path}],
       archives=[],
-      files=[{'value': '/user/' + DEFAULT_USER.get() + '/common/extract_archive_in_hdfs.sh'}, {"value": upload_path + '/' + urllib.quote(file_name)}],
+      files=[{'value': '/user/' + DEFAULT_USER.get() + '/common/extract_archive_in_hdfs.sh'}, {"value": upload_path + '/' + urllib.parse.quote(file_name)}],
       env_var=[{'value': 'HADOOP_USER_NAME=${wf:user()}'}],
       last_executed=start_time
   )
@@ -54,9 +56,9 @@ def extract_archive_in_hdfs(request, upload_path, file_name):
 def _upload_extract_archive_script_to_hdfs(fs):
   if not fs.exists('/user/' + DEFAULT_USER.get() + '/common/'):
     fs.do_as_user(DEFAULT_USER.get(), fs.mkdir, '/user/' + DEFAULT_USER.get() + '/common/')
-    fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0755)
+    fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0o755)
 
   if not fs.do_as_user(DEFAULT_USER.get(), fs.exists, '/user/' + DEFAULT_USER.get() + '/common/extract_archive_in_hdfs.sh'):
     fs.do_as_user(DEFAULT_USER.get(), fs.copyFromLocal, get_desktop_root() + '/core/src/desktop/lib/tasks/extract_archive/extract_in_hdfs.sh',
                           '/user/' + DEFAULT_USER.get() + '/common/extract_archive_in_hdfs.sh')
-    fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0755)
+    fs.do_as_user(DEFAULT_USER.get(), fs.chmod, '/user/' + DEFAULT_USER.get() + '/common/', 0o755)

+ 8 - 2
desktop/core/src/desktop/lib/test_export_csvxls.py

@@ -16,13 +16,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import StringIO
+from future import standard_library
+standard_library.install_aliases()
+import sys
 
 from nose.tools import assert_equal
 from openpyxl import load_workbook
 
 from desktop.lib.export_csvxls import create_generator, make_response
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from cStringIO import StringIO as string_io
 
 def content_generator(header, data):
   yield header, data
@@ -80,7 +86,7 @@ def test_export_xls():
 def _read_xls_sheet_data(response):
   content = ''.join(response.content)
 
-  data = StringIO.StringIO()
+  data = string_io()
   data.write(content)
 
   wb = load_workbook(filename=data, read_only=True)

+ 1 - 0
desktop/core/src/desktop/lib/test_runners.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import sys
 import threading
 

+ 1 - 0
desktop/core/src/desktop/lib/test_utils.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from past.builtins import basestring
 import json
 from lxml import objectify, etree
 

+ 10 - 4
desktop/core/src/desktop/lib/thread_util.py

@@ -15,13 +15,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+from future import standard_library
+standard_library.install_aliases()
 import logging
 import socket
 import sys
 import threading
 import traceback
-import StringIO
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from cStringIO import StringIO as string_io
 
 LOG = logging.getLogger(__name__)
 
@@ -42,10 +48,10 @@ def dump_traceback(file=sys.stderr, all_threads=True):
     else:
       name = "Thread"
 
-    trace_buffer = StringIO.StringIO()
-    print >> trace_buffer, "%s: %s %s %s (most recent call last):" % (socket.gethostname(), name, thread.name, thread.ident)
+    trace_buffer = string_io()
+    print("%s: %s %s %s (most recent call last):" % (socket.gethostname(), name, thread.name, thread.ident), file=trace_buffer)
     frame = sys._current_frames()[thread.ident]
     traceback.print_stack(frame, file=trace_buffer)
 
-    print >> file, trace_buffer.getvalue()
+    print(trace_buffer.getvalue(), file=file)
     logging.debug(trace_buffer.getvalue())

+ 10 - 3
desktop/core/src/desktop/lib/thread_util_test.py

@@ -15,13 +15,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import StringIO
+from future import standard_library
+standard_library.install_aliases()
+import sys
 import threading
 import time
 
 from nose.tools import assert_true
 from desktop.lib.thread_util import dump_traceback
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from cStringIO import StringIO as string_io
+
 def test_dump_traceback():
   started = threading.Event()
   stopped = threading.Event()
@@ -42,12 +49,12 @@ def test_dump_traceback():
     started.wait(10.0)
     assert_true(started.is_set())
 
-    out = StringIO.StringIO()
+    out = string_io()
     dump_traceback(file=out)
 
     assert_true(header in out.getvalue())
 
-    out = StringIO.StringIO()
+    out = string_io()
     dump_traceback(file=out, all_threads=False)
 
     assert_true(header not in out.getvalue())

+ 2 - 1
desktop/core/src/desktop/lib/thrift_/TSSLSocketWithWildcardSAN.py

@@ -15,6 +15,7 @@
 
 # This file comes from the Apache2 licenced file source code file https://github.com/apache/incubator-impala/blob/45ff0f9e674f54b35afb2b5eced0d6ec346890d6/shell/TSSLSocketWithWildcardSAN.py
 
+from builtins import map
 import re
 
 from thrift.transport import TSSLSocket
@@ -54,7 +55,7 @@ class TSSLSocketWithWildcardSAN(TSSLSocket.TSSLSocket):
       self._match_hostname(cert, self.host)
       self.is_valid = True
       return
-    except CertificateError, ce:
+    except CertificateError as ce:
       raise TTransportException(
         type=TTransportException.UNKNOWN,
         message='Certificate error with remote host: %s' % (ce))

+ 9 - 4
desktop/core/src/desktop/lib/thrift_/http_client.py

@@ -16,15 +16,20 @@
 # limitations under the License.
 #
 
+from future import standard_library
+standard_library.install_aliases()
 import logging
-
-from cStringIO import StringIO
+import sys
 
 from thrift.transport.TTransport import *
 
 from desktop.lib.rest.http_client import HttpClient
 from desktop.lib.rest.resource import Resource
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from cStringIO import StringIO as string_io
 
 LOG = logging.getLogger(__name__)
 
@@ -45,7 +50,7 @@ class THttpClient(TTransportBase):
     self._client = HttpClient(self._base_url, logger=LOG)
     self._data = None
     self._headers = None
-    self._wbuf = StringIO()
+    self._wbuf = string_io()
 
   def open(self):
     pass
@@ -80,7 +85,7 @@ class THttpClient(TTransportBase):
 
   def flush(self):
     data = self._wbuf.getvalue()
-    self._wbuf = StringIO()
+    self._wbuf = string_io()
 
     # POST
     self._root = Resource(self._client)

+ 13 - 6
desktop/core/src/desktop/lib/thrift_sasl.py

@@ -18,12 +18,19 @@
 #
 """ SASL transports for Thrift. """
 
-from cStringIO import StringIO
+from future import standard_library
+standard_library.install_aliases()
 from thrift.transport import TTransport
 from thrift.transport.TTransport import *
 from thrift.protocol import TBinaryProtocol
 import sasl
 import struct
+import sys
+
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from cStringIO import StringIO as string_io
 
 class TSaslClientTransport(TTransportBase, CReadableTransport):
   START = 1
@@ -42,8 +49,8 @@ class TSaslClientTransport(TTransportBase, CReadableTransport):
     self.sasl_client_factory = sasl_client_factory
     self.sasl = None
     self.mechanism = mechanism
-    self.__wbuf = StringIO()
-    self.__rbuf = StringIO()
+    self.__wbuf = string_io()
+    self.__rbuf = string_io()
     self.opened = False
     self.encode = None
 
@@ -123,7 +130,7 @@ class TSaslClientTransport(TTransportBase, CReadableTransport):
       self._flushPlain(buffer)
 
     self._trans.flush()
-    self.__wbuf = StringIO()
+    self.__wbuf = string_io()
 
   def _flushEncoded(self, buffer):
     # sasl.ecnode() does the encoding and adds the length header, so nothing
@@ -168,7 +175,7 @@ class TSaslClientTransport(TTransportBase, CReadableTransport):
     else:
       # If the frames are not encoded, just pass it through
       decoded = self._trans.readAll(length)
-    self.__rbuf = StringIO(decoded)
+    self.__rbuf = string_io(decoded)
 
   def close(self):
     self._trans.close()
@@ -187,5 +194,5 @@ class TSaslClientTransport(TTransportBase, CReadableTransport):
     while len(prefix) < reqlen:
       self._read_frame()
       prefix += self.__rbuf.getvalue()
-    self.__rbuf = StringIO(prefix)
+    self.__rbuf = string_io(prefix)
     return self.__rbuf

+ 32 - 22
desktop/core/src/desktop/lib/thrift_util.py

@@ -15,8 +15,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from builtins import map
+from builtins import range
+from past.builtins import basestring
+from past.utils import old_div
+from builtins import object
 import base64
-import Queue
+import queue
 import logging
 import socket
 import threading
@@ -44,6 +52,8 @@ from desktop.lib.thrift_.TSSLSocketWithWildcardSAN import TSSLSocketWithWildcard
 from desktop.lib.thrift_sasl import TSaslClientTransport
 from desktop.lib.exceptions import StructuredException, StructuredThriftTransportException
 
+if sys.version_info[0] > 2:
+  from past.builtins import long
 
 LOG = logging.getLogger(__name__)
 
@@ -54,7 +64,7 @@ LOG = logging.getLogger(__name__)
 MAX_RECURSION_DEPTH = 50
 
 
-class LifoQueue(Queue.Queue):
+class LifoQueue(queue.Queue):
     '''
     Variant of Queue that retrieves most recently added entries first.
 
@@ -202,7 +212,7 @@ class ConnectionPooler(object):
       try:
         if _get_pool_key(conf) not in self.pooldict:
           q = LifoQueue(self.poolsize)
-          for i in xrange(self.poolsize):
+          for i in range(self.poolsize):
             client = construct_superclient(conf)
             client.CID = i
             q.put(client, False)
@@ -244,7 +254,7 @@ class ConnectionPooler(object):
           duration = time.time() - start_pool_get_time
           message = "Thrift client %s got connection %s after %.2f seconds" % (self, connection.CID, duration)
           log_if_slow_call(duration=duration, message=message)
-      except Queue.Empty:
+      except queue.Empty:
         has_waited_for = time.time() - start_pool_get_time
         if get_client_timeout is not None and has_waited_for > get_client_timeout:
           raise socket.timeout(
@@ -405,20 +415,20 @@ class PooledClient(object):
 
           superclient.set_timeout(self.conf.timeout_seconds)
           return attr(*args, **kwargs)
-        except TApplicationException, e:
+        except TApplicationException as e:
           # Unknown thrift exception... typically IO errors
           logging.info("Thrift saw an application exception: " + str(e), exc_info=False)
           raise StructuredException('THRIFTAPPLICATION', str(e), data=None, error_code=502)
-        except socket.error, e:
+        except socket.error as e:
           logging.info("Thrift saw a socket error: " + str(e), exc_info=False)
           raise StructuredException('THRIFTSOCKET', str(e), data=None, error_code=502)
-        except TTransportException, e:
+        except TTransportException as e:
           err_msg = str(e)
           logging.info("Thrift saw a transport exception: " + err_msg, exc_info=False)
           if err_msg and 'generic failure: Unable to find a callback: 32775' in err_msg:
             raise StructuredException(_("Increase the sasl_max_buffer value in hue.ini"), err_msg, data=None, error_code=502)
           raise StructuredThriftTransportException(e, error_code=502)
-        except Exception, e:
+        except Exception as e:
           # Stack tends to be only noisy here.
           logging.info("Thrift saw exception: " + str(e), exc_info=False)
           raise
@@ -489,11 +499,11 @@ class SuperClient(object):
           log_if_slow_call(duration=duration, message=logmsg)
 
           return ret
-        except socket.error, e:
+        except socket.error as e:
           pass
-        except TTransportException, e:
+        except TTransportException as e:
           pass
-        except Exception, e:
+        except Exception as e:
           logging.exception("Thrift saw exception (this may be expected).")
           raise
 
@@ -533,7 +543,7 @@ def _unpack_guid_secret_in_handle(str_args):
 
         str_args = str_args.replace(secret.group(1), unpack_guid(encoded_secret))
         str_args = str_args.replace(guid.group(1), unpack_guid(encoded_guid))
-      except Exception, e:
+      except Exception as e:
         logging.warn("Unable to unpack the secret and guid in Thrift Handle: %s" % e)
 
   return str_args
@@ -555,7 +565,7 @@ def simpler_string(thrift_obj):
   TODO(philip): Use this in SuperClient, above.
   """
   L = []
-  for key, value in thrift_obj.__dict__.iteritems():
+  for key, value in thrift_obj.__dict__.items():
     if value is None:
       continue
     if hasattr(value, "thrift_spec"):
@@ -599,11 +609,11 @@ def thrift2json(tft):
   """
   if isinstance(tft,type(None)):
     return None
-  if isinstance(tft,(float,int,long,complex,basestring)):
+  if isinstance(tft,(float,int,complex,basestring)):
     return tft
   if isinstance(tft,dict):
     d = {}
-    for key, val in tft.iteritems():
+    for key, val in tft.items():
       d[key] = thrift2json(val)
     return d
   if isinstance(tft,list):
@@ -713,7 +723,7 @@ def _jsonable2thrift_helper(jsonable, type_enum, spec_args, default, recursion_d
     check_type(jsonable, dict)
     key_type_enum, key_spec_args, val_type_enum, val_spec_args = spec_args
     out = dict()
-    for k_jsonable, v_jsonable in jsonable.iteritems():
+    for k_jsonable, v_jsonable in jsonable.items():
       k = _jsonable2thrift_helper(k_jsonable, key_type_enum, key_spec_args, None, recursion_depth + 1)
       v = _jsonable2thrift_helper(v_jsonable, val_type_enum, val_spec_args, None, recursion_depth + 1)
       out[k] = v
@@ -724,7 +734,7 @@ def _jsonable2thrift_helper(jsonable, type_enum, spec_args, default, recursion_d
     # as a map with values True.
     set_type_enum, set_spec_args = spec_args
     out = set()
-    for k, v in jsonable.iteritems():
+    for k, v in jsonable.items():
       assert v is True, "Expected set value to be True.  Got: %s" % repr(v)
       out.add(_jsonable2thrift_helper(k, set_type_enum, set_spec_args, None, recursion_depth + 1))
     return out
@@ -770,8 +780,8 @@ def enum_as_sequence(enum):
   Arguments:
   - `enum`: The class of a Thrift-generated enum
   """
-  return filter(lambda x: not x.startswith("__")
-                and  x not in ["_VALUES_TO_NAMES", "_NAMES_TO_VALUES"],dir(enum))
+  return [x for x in dir(enum) if not x.startswith("__")
+                and  x not in ["_VALUES_TO_NAMES", "_NAMES_TO_VALUES", "next"]]
 
 def fixup_enums(obj, name_class_map, suffix="AsString"):
   """
@@ -783,7 +793,7 @@ def fixup_enums(obj, name_class_map, suffix="AsString"):
 
   This is destructive - it uses setattr.
   """
-  for n in name_class_map.keys():
+  for n in list(name_class_map.keys()):
     c = name_class_map[n]
     setattr(obj, n + suffix, c._VALUES_TO_NAMES[getattr(obj,n)])
   return obj
@@ -794,9 +804,9 @@ def is_thrift_struct(o):
 
 # Same in resource.py for not losing the trace class
 def log_if_slow_call(duration, message):
-  if duration >= WARN_LEVEL_CALL_DURATION_MS / 1000:
+  if duration >= old_div(WARN_LEVEL_CALL_DURATION_MS, 1000):
     LOG.warn('SLOW: %.2f - %s' % (duration, message))
-  elif duration >= INFO_LEVEL_CALL_DURATION_MS / 1000:
+  elif duration >= old_div(INFO_LEVEL_CALL_DURATION_MS, 1000):
     LOG.info('SLOW: %.2f - %s' % (duration, message))
   else:
     LOG.debug(message)

+ 6 - 4
desktop/core/src/desktop/lib/thrift_util_test.py

@@ -14,6 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import
+from builtins import range
+from builtins import object
 import logging
 import os
 import socket
@@ -31,9 +34,8 @@ if not gen_py_path in sys.path:
 from djangothrift_test_gen.ttypes import TestStruct, TestNesting, TestEnum, TestManyTypes
 from djangothrift_test_gen import TestService
 
-import python_util
-import thrift_util
-from thrift_util import jsonable2thrift, thrift2json, _unpack_guid_secret_in_handle
+from desktop.lib import python_util, thrift_util
+from desktop.lib.thrift_util import jsonable2thrift, thrift2json, _unpack_guid_secret_in_handle
 
 from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory
 from thrift.server import TServer
@@ -112,7 +114,7 @@ class SimpleThriftServer(object):
     try:
       logging.info("Stopping SimpleThriftServer (pid %s)" % (self.pid,))
       os.kill(self.pid, 15)
-    except Exception, ex:
+    except Exception as ex:
       logging.exception("(Potentially ok) Exception while stopping server")
     os.waitpid(self.pid, 0)
     self.pid = 0

+ 1 - 0
desktop/core/src/desktop/lib/vcs/apis/base_api.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import logging
 
 from django.utils.translation import ugettext as _

+ 16 - 8
desktop/core/src/desktop/lib/vcs/apis/github_readonly_api.py

@@ -16,11 +16,12 @@
 # limitations under the License.
 
 
+from future import standard_library
+standard_library.install_aliases()
 import binascii
 import logging
 import re
-import urllib
-import urlparse
+import sys
 
 from django.http import HttpResponseBadRequest
 from django.utils.translation import ugettext as _
@@ -33,6 +34,13 @@ from desktop.conf import VCS
 from desktop.lib.vcs.apis.base_api import Api, GIT_READ_ONLY
 from desktop.lib.vcs.github_client import GithubClientException
 
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+  from urllib.parse import unquote as urllib_unquote, urlsplit as lib_urlsplit, urlunsplit as lib_urlunsplit
+else:
+  from urllib import unquote as urllib_unquote
+  from urlparse import urlsplit as lib_urlsplit, urlunsplit as lib_urlunsplit
+
 LOG = logging.getLogger(__name__)
 
 
@@ -74,9 +82,9 @@ class GithubReadOnlyApi(Api):
         try:
           response['content'] = blob['content'].decode('base64')
           response['status'] = 0
-        except binascii.Error, e:
+        except binascii.Error as e:
           raise GithubClientException('Failed to decode file contents, check if file content is properly base64-encoded: %s' % e)
-        except KeyError, e:
+        except KeyError as e:
           raise GithubClientException('Failed to find expected content object in blob object: %s' % e)
     else:
       return HttpResponseBadRequest(_('url param is required'))
@@ -102,18 +110,18 @@ class GithubReadOnlyApi(Api):
     return re.compile('%s/%s/%s/tree/%s' % (self._get_base_url(), self.OWNER_RE, self.REPO_RE, self.BRANCH_RE))
 
   def _get_base_url(self):
-    split_url = urlparse.urlsplit(self._remote_url)
-    return urlparse.urlunsplit((split_url.scheme, split_url.netloc, '', "", ""))
+    split_url = lib_urlsplit(self._remote_url)
+    return lib_urlunsplit((split_url.scheme, split_url.netloc, '', "", ""))
 
   def _clean_path(self, filepath):
     cleaned_path = filepath.strip('/')
-    cleaned_path = urllib.unquote(cleaned_path)
+    cleaned_path = urllib_unquote(cleaned_path)
     return cleaned_path
 
   def _get_contents(self, owner, repo, path):
     try:
       return self._root.get('repos/%s/%s/contents/%s' % (owner, repo, path))
-    except RestException, e:
+    except RestException as e:
       raise GithubClientException('Could not find GitHub object, check owner, repo or path: %s' % e)
 
 

+ 15 - 7
desktop/core/src/desktop/lib/vcs/github_client.py

@@ -16,11 +16,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
 import binascii
 import json
 import logging
 import re
-import urllib
+import sys
 
 from desktop.lib.rest.http_client import HttpClient, RestException
 from desktop.lib.rest import resource
@@ -28,6 +31,11 @@ from desktop.lib.rest import resource
 from desktop.conf import VCS
 from desktop.lib.vcs.apis.base_api import GITHUB_OFFICIAL
 
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+  from urllib.parse import unquote as urllib_unquote
+else:
+  from urllib import unquote as urllib_unquote
 
 LOG = logging.getLogger(__name__)
 
@@ -97,7 +105,7 @@ class GithubClient(object):
       response = root.post('login/oauth/access_token', headers=headers, data=json.dumps(data))
       result = cls._get_json(response)
       return result['access_token']
-    except RestException, e:
+    except RestException as e:
       raise GithubClientException('Failed to request access token from GitHub: %s' % e)
     except KeyError:
       raise GithubClientException('Failed to find access_token in GitHub oAuth response')
@@ -136,7 +144,7 @@ class GithubClient(object):
 
   def _clean_path(self, filepath):
     cleaned_path = filepath.strip('/')
-    cleaned_path = urllib.unquote(cleaned_path)
+    cleaned_path = urllib_unquote(cleaned_path)
     return cleaned_path
 
 
@@ -160,9 +168,9 @@ class GithubClient(object):
       blob = self.get_blob(owner, repo, sha)
       content = blob['content'].decode('base64')
       return content
-    except binascii.Error, e:
+    except binascii.Error as e:
       raise GithubClientException('Failed to decode file contents, check if file content is properly base64-encoded: %s' % e)
-    except KeyError, e:
+    except KeyError as e:
       raise GithubClientException('Failed to find expected content object in blob object: %s' % e)
 
 
@@ -199,7 +207,7 @@ class GithubClient(object):
         )
       response = self._root.get('repos/%s/%s/git/trees/%s' % (owner, repo, sha), headers=self.__headers, params=self.__params)
       return self._get_json(response)
-    except RestException, e:
+    except RestException as e:
       raise GithubClientException('Could not find GitHub object, check owner, repo and filepath or permissions: %s' % e)
 
 
@@ -211,5 +219,5 @@ class GithubClient(object):
     try:
       response = self._root.get('repos/%s/%s/git/blobs/%s' % (owner, repo, sha), headers=self.__headers, params=self.__params)
       return self._get_json(response)
-    except RestException, e:
+    except RestException as e:
       raise GithubClientException('Could not find GitHub object, check owner, repo and sha or permissions: %s' % e)

+ 3 - 1
desktop/core/src/desktop/lib/view_util.py

@@ -15,7 +15,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Utilities for views (text and number formatting, etc)"""
+from __future__ import division
 
+from past.utils import old_div
 import datetime
 import logging
 import math
@@ -42,7 +44,7 @@ def big_filesizeformat(bytes):
   index = int(math.floor(math.log(bytes, 1024)))
   index = min(len(units) - 1, index)
 
-  return( "%.1f %s" % (bytes / math.pow(1024, index), units[index]) )
+  return( "%.1f %s" % (old_div(bytes, math.pow(1024, index)), units[index]) )
 
 def format_time_diff(start=None, end=None):
   """

+ 3 - 1
desktop/core/src/desktop/lib/view_util_test.py

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import division
+from past.utils import old_div
 from nose.tools import *
 
 from desktop.lib.view_util import big_filesizeformat, format_time_diff, format_duration_in_millis
@@ -30,7 +32,7 @@ def test_big_filesizeformat():
   assert_equal("1.0 MB", big_filesizeformat(1024*1024))
   assert_equal("1.1 GB", big_filesizeformat(int(1.1*1024*1024*1024)))
   assert_equal("2.0 TB", big_filesizeformat(2*1024*1024*1024*1024))
-  assert_equal("1.5 PB", big_filesizeformat(3*1024*1024*1024*1024*1024/2))
+  assert_equal("1.5 PB", big_filesizeformat(old_div(3*1024*1024*1024*1024*1024,2)))
 
 def test_format_time_diff():
   assert_equal("1h:0m:0s", format_time_diff(datetime.datetime.fromtimestamp(0), datetime.datetime.fromtimestamp(60*60*1)))

+ 60 - 47
desktop/core/src/desktop/lib/wsgiserver.py

@@ -102,17 +102,23 @@ number of requests and their responses, so we run a nested loop:
 """
 
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import hex
+from builtins import range
+from past.builtins import basestring
+from builtins import object
 import base64
 import os
-import Queue
+import queue
 import re
 quoted_slash = re.compile("(?i)%2F")
 import rfc822
 import socket
 try:
-    import cStringIO as StringIO
+    import io as StringIO
 except ImportError:
-    import StringIO
+    import io
 
 _fileobject_uses_str_type = isinstance(socket._fileobject(None)._rbuf, basestring)
 
@@ -120,8 +126,6 @@ import sys
 import threading
 import time
 import traceback
-from urllib import unquote
-from urlparse import urlparse
 import warnings
 
 try:
@@ -133,6 +137,15 @@ except ImportError:
 import errno
 import logging
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+  import urllib.request, urllib.error
+  from urllib.parse import unquote as urllib_unquote, urlparse as lib_urlparse
+else:
+  from cStringIO import StringIO as string_io
+  from urllib import unquote as urllib_unquote
+  from urlparse import urlparse as lib_urlparse
+
 LOG = logging.getLogger(__name__)
 
 def plat_specific_errors(*errnames):
@@ -145,7 +158,7 @@ def plat_specific_errors(*errnames):
     errno_names = dir(errno)
     nums = [getattr(errno, k) for k in errnames if k in errno_names]
     # de-dupe the list
-    return dict.fromkeys(nums).keys()
+    return list(dict.fromkeys(nums).keys())
 
 socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR")
 
@@ -181,7 +194,7 @@ class WSGIPathInfoDispatcher(object):
     
     def __init__(self, apps):
         try:
-            apps = apps.items()
+            apps = list(apps.items())
         except AttributeError:
             pass
         
@@ -267,8 +280,8 @@ class SizeCheckWrapper(object):
     def __iter__(self):
         return self
     
-    def next(self):
-        data = self.rfile.next()
+    def __next__(self):
+        data = next(self.rfile)
         self.bytes_read += len(data)
         self._check_length()
         return data
@@ -376,7 +389,7 @@ class HTTPRequest(object):
         environ["REQUEST_METHOD"] = method
         
         # path may be an abs_path (including "http://host.domain.tld");
-        scheme, location, path, params, qs, frag = urlparse(path)
+        scheme, location, path, params, qs, frag = lib_urlparse(path)
         
         if frag:
             self.simple_response("400 Bad Request",
@@ -396,7 +409,7 @@ class HTTPRequest(object):
         # But note that "...a URI must be separated into its components
         # before the escaped characters within those components can be
         # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2
-        atoms = [unquote(x) for x in quoted_slash.split(path)]
+        atoms = [urllib_unquote(x) for x in quoted_slash.split(path)]
         path = "%2F".join(atoms)
         environ["PATH_INFO"] = path
         
@@ -433,7 +446,7 @@ class HTTPRequest(object):
         # then all the http headers
         try:
             self.read_headers()
-        except ValueError, ex:
+        except ValueError as ex:
             self.simple_response("400 Bad Request", repr(ex.args))
             return
         
@@ -532,7 +545,7 @@ class HTTPRequest(object):
     def decode_chunked(self):
         """Decode the 'chunked' transfer coding."""
         cl = 0
-        data = StringIO.StringIO()
+        data = string_io()
         while True:
             line = self.rfile.readline().strip().split(";", 1)
             chunk_size = int(line.pop(0), 16)
@@ -624,7 +637,7 @@ class HTTPRequest(object):
         
         try:
             self.wfile.sendall("".join(buf))
-        except socket.error, x:
+        except socket.error as x:
             if x.args[0] not in socket_errors_to_ignore:
                 raise
     
@@ -760,7 +773,7 @@ if not _fileobject_uses_str_type:
                 try:
                     bytes_sent = self.send(data)
                     data = data[bytes_sent:]
-                except socket.error, e:
+                except socket.error as e:
                     if e.args[0] not in socket_errors_nonblocking:
                         raise
 
@@ -777,7 +790,7 @@ if not _fileobject_uses_str_type:
             while True:
                 try:
                     return self._sock.recv(size)
-                except socket.error, e:
+                except socket.error as e:
                     if (e.args[0] not in socket_errors_nonblocking
                         and e.args[0] not in socket_error_eintr):
                         raise
@@ -794,7 +807,7 @@ if not _fileobject_uses_str_type:
             buf.seek(0, 2)  # seek end
             if size < 0:
                 # Read until EOF
-                self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
+                self._rbuf = string_io()  # reset _rbuf.  we consume it via buf.
                 while True:
                     data = self.recv(rbufsize)
                     if not data:
@@ -808,11 +821,11 @@ if not _fileobject_uses_str_type:
                     # Already have size bytes in our buffer?  Extract and return.
                     buf.seek(0)
                     rv = buf.read(size)
-                    self._rbuf = StringIO.StringIO()
+                    self._rbuf = string_io()
                     self._rbuf.write(buf.read())
                     return rv
 
-                self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
+                self._rbuf = string_io()  # reset _rbuf.  we consume it via buf.
                 while True:
                     left = size - buf_len
                     # recv() will malloc the amount of memory given as its
@@ -858,7 +871,7 @@ if not _fileobject_uses_str_type:
                 buf.seek(0)
                 bline = buf.readline(size)
                 if bline.endswith('\n') or len(bline) == size:
-                    self._rbuf = StringIO.StringIO()
+                    self._rbuf = string_io()
                     self._rbuf.write(buf.read())
                     return bline
                 del bline
@@ -868,7 +881,7 @@ if not _fileobject_uses_str_type:
                     # Speed up unbuffered case
                     buf.seek(0)
                     buffers = [buf.read()]
-                    self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
+                    self._rbuf = string_io()  # reset _rbuf.  we consume it via buf.
                     data = None
                     recv = self.recv
                     while data != "\n":
@@ -879,7 +892,7 @@ if not _fileobject_uses_str_type:
                     return "".join(buffers)
 
                 buf.seek(0, 2)  # seek end
-                self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
+                self._rbuf = string_io()  # reset _rbuf.  we consume it via buf.
                 while True:
                     data = self.recv(self._rbufsize)
                     if not data:
@@ -900,10 +913,10 @@ if not _fileobject_uses_str_type:
                 if buf_len >= size:
                     buf.seek(0)
                     rv = buf.read(size)
-                    self._rbuf = StringIO.StringIO()
+                    self._rbuf = string_io()
                     self._rbuf.write(buf.read())
                     return rv
-                self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
+                self._rbuf = string_io()  # reset _rbuf.  we consume it via buf.
                 while True:
                     data = self.recv(self._rbufsize)
                     if not data:
@@ -946,7 +959,7 @@ else:
                 try:
                     bytes_sent = self.send(data)
                     data = data[bytes_sent:]
-                except socket.error, e:
+                except socket.error as e:
                     if e.args[0] not in socket_errors_nonblocking:
                         raise
 
@@ -963,7 +976,7 @@ else:
             while True:
                 try:
                     return self._sock.recv(size)
-                except socket.error, e:
+                except socket.error as e:
                     if (e.args[0] not in socket_errors_nonblocking
                         and e.args[0] not in socket_error_eintr):
                         raise
@@ -1105,7 +1118,7 @@ class SSL_fileobject(CP_fileobject):
                 time.sleep(self.ssl_retry)
             except SSL.WantWriteError:
                 time.sleep(self.ssl_retry)
-            except SSL.SysCallError, e:
+            except SSL.SysCallError as e:
                 if is_reader and e.args == (-1, 'Unexpected EOF'):
                     return ""
                 
@@ -1113,7 +1126,7 @@ class SSL_fileobject(CP_fileobject):
                 if is_reader and errnum in socket_errors_to_ignore:
                     return ""
                 raise socket.error(errnum)
-            except SSL.Error, e:
+            except SSL.Error as e:
                 if is_reader and e.args == (-1, 'Unexpected EOF'):
                     return ""
                 
@@ -1216,7 +1229,7 @@ class HTTPConnection(object):
                 if req.close_connection:
                     return
         
-        except socket.error, e:
+        except socket.error as e:
             errnum = e.args[0]
             if errnum == 'timed out':
                 # Don't send a 408 if there is no outstanding request; only
@@ -1231,7 +1244,7 @@ class HTTPConnection(object):
             return
         except (KeyboardInterrupt, SystemExit):
             raise
-        except FatalSSLAlert, e:
+        except FatalSSLAlert as e:
             # Close the connection.
             return
         except NoSSLError:
@@ -1242,7 +1255,7 @@ class HTTPConnection(object):
                     "The client sent a plain HTTP request, but "
                     "this server only speaks HTTPS on this port.")
                 self.linger = True
-        except Exception, e:
+        except Exception as e:
             if req and not req.sent_headers:
                 req.simple_response("500 Internal Server Error", format_exc())
     
@@ -1317,9 +1330,9 @@ class WorkerThread(threading.Thread):
                     finally:
                         conn.close()
                         self.conn = None
-                except Exception, ex:
+                except Exception as ex:
                     LOG.exception('WSGI (%s) error: %s' % (self, ex))
-        except (KeyboardInterrupt, SystemExit), exc:
+        except (KeyboardInterrupt, SystemExit) as exc:
             self.server.interrupt = exc
             return
 
@@ -1336,12 +1349,12 @@ class ThreadPool(object):
         self.min = min
         self.max = max
         self._threads = []
-        self._queue = Queue.Queue()
+        self._queue = queue.Queue()
         self.get = self._queue.get
     
     def start(self):
         """Start the pool of threads."""
-        for i in xrange(self.min):
+        for i in range(self.min):
             self._threads.append(WorkerThread(self.server))
         for worker in self._threads:
             worker.setName("CP WSGIServer " + worker.getName())
@@ -1362,7 +1375,7 @@ class ThreadPool(object):
     
     def grow(self, amount):
         """Spawn new worker threads (not above self.max)."""
-        for i in xrange(amount):
+        for i in range(amount):
             if self.max > 0 and len(self._threads) >= self.max:
                 break
             worker = WorkerThread(self.server)
@@ -1380,7 +1393,7 @@ class ThreadPool(object):
                 amount -= 1
         
         if amount > 0:
-            for i in xrange(min(amount, len(self._threads) - self.min)):
+            for i in range(min(amount, len(self._threads) - self.min)):
                 # Put a number of shutdown requests on the queue equal
                 # to 'amount'. Once each of those is processed by a worker,
                 # that worker will terminate and be culled from our list
@@ -1417,12 +1430,12 @@ class ThreadPool(object):
                 except (AssertionError,
                         # Ignore repeated Ctrl-C.
                         # See http://www.cherrypy.org/ticket/691.
-                        KeyboardInterrupt), exc1:
+                        KeyboardInterrupt) as exc1:
                     pass
 
 
 
-class SSLConnection:
+class SSLConnection(object):
     """A thread-safe wrapper for an SSL.Connection.
     
     *args: the arguments to create the wrapped SSL.Connection(*args).
@@ -1440,13 +1453,13 @@ class SSLConnection:
               'sock_shutdown', 'get_peer_certificate', 'want_read',
               'want_write', 'set_connect_state', 'set_accept_state',
               'connect_ex', 'sendall', 'settimeout'):
-        exec """def %s(self, *args):
+        exec("""def %s(self, *args):
         self._lock.acquire()
         try:
             return self._ssl_conn.%s(*args)
         finally:
             self._lock.release()
-""" % (f, f)
+""" % (f, f))
 
 
 try:
@@ -1626,7 +1639,7 @@ class CherryPyWSGIServer(object):
             
             # So everyone can access the socket...
             try:
-              os.chmod(self.bind_addr, 0777)
+              os.chmod(self.bind_addr, 0o777)
             except IOError:
               pass
             
@@ -1648,14 +1661,14 @@ class CherryPyWSGIServer(object):
             af, socktype, proto, canonname, sa = res
             try:
                 self._bind(af, socktype, proto)
-            except socket.error, msg:
+            except socket.error as msg:
                 if self.socket:
                     self.socket.close()
                 self.socket = None
                 continue
             break
         if not self.socket:
-            raise socket.error, msg
+            raise socket.error(msg)
 
     def listen_and_loop(self):
         """
@@ -1702,7 +1715,7 @@ class CherryPyWSGIServer(object):
               ctx.use_certificate_file(self.ssl_certificate)
               if self.ssl_certificate_chain:
                 ctx.use_certificate_chain_file(self.ssl_certificate_chain)
-            except Exception, ex:
+            except Exception as ex:
               logging.exception('SSL key and certificate could not be found or have a problem')
               raise ex
             ctx.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
@@ -1761,7 +1774,7 @@ class CherryPyWSGIServer(object):
             # notice keyboard interrupts on Win32, which don't interrupt
             # accept() by default
             return
-        except socket.error, x:
+        except socket.error as x:
             if x.args[0] in socket_error_eintr:
                 # I *think* this is right. EINTR should occur when a signal
                 # is received during the accept() call; all docs say retry
@@ -1798,7 +1811,7 @@ class CherryPyWSGIServer(object):
                 # Touch our own socket to make accept() return immediately.
                 try:
                     host, port = sock.getsockname()[:2]
-                except socket.error, x:
+                except socket.error as x:
                     if x.args[0] not in socket_errors_to_ignore:
                         raise
                 else:

+ 17 - 10
desktop/core/src/desktop/log/__init__.py

@@ -15,6 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
 import logging
 import logging.config
 import os
@@ -22,7 +26,6 @@ import os.path
 import re
 import sys
 
-from cStringIO import StringIO
 from logging import FileHandler
 from logging.handlers import RotatingFileHandler
 
@@ -30,6 +33,10 @@ from desktop.lib.paths import get_desktop_root
 from desktop.log import formatter
 from desktop.log.formatter import MessageOnlyFormatter
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from cStringIO import StringIO as string_io
 
 DEFAULT_LOG_DIR = 'logs'
 LOG_FORMAT = '[%(asctime)s] %(module)-12s %(levelname)-8s %(message)s'
@@ -59,10 +66,10 @@ def _read_log_conf(proc_name, log_dir):
 
   try:
     raw = file(log_conf).read()
-    sio = StringIO(CONF_RE.sub(_repl, raw))
+    sio = string_io(CONF_RE.sub(_repl, raw))
     return sio
-  except IOError, ex:
-    print >> sys.stderr, "ERROR: Failed to open %s: %s" % (log_conf, ex)
+  except IOError as ex:
+    print("ERROR: Failed to open %s: %s" % (log_conf, ex), file=sys.stderr)
     return None
 
 
@@ -82,7 +89,7 @@ def get_audit_logger():
   from desktop.conf import AUDIT_EVENT_LOG_DIR, AUDIT_LOG_MAX_FILE_SIZE
 
   audit_logger = logging.getLogger('audit')
-  if not filter(lambda hclass: isinstance(hclass, AuditHandler), audit_logger.handlers): # Don't add handler twice
+  if not [hclass for hclass in audit_logger.handlers if isinstance(hclass, AuditHandler)]: # Don't add handler twice
     size, unit = int(AUDIT_LOG_MAX_FILE_SIZE.get()[:-2]), AUDIT_LOG_MAX_FILE_SIZE.get()[-2:]
     maxBytes = size * 1024 ** (1 if unit == 'KB' else 2 if unit == 'MB' else 3)
 
@@ -107,8 +114,8 @@ def chown_log_dir(uid, gid):
     for entry in os.listdir(_log_dir):
       os.chown(os.path.join(_log_dir, entry), uid, gid)
     return True
-  except OSError, ex:
-    print >> sys.stderr, 'Failed to chown log directory %s: ex' % (_log_dir, ex)
+  except OSError as ex:
+    print('Failed to chown log directory %s: ex' % (_log_dir, ex), file=sys.stderr)
     return False
 
 
@@ -135,8 +142,8 @@ def basic_logging(proc_name, log_dir=None):
   if not os.path.exists(log_dir):
     try:
       os.makedirs(log_dir)
-    except OSError, err:
-      print >> sys.stderr, 'Failed to create log directory "%s": %s' % (log_dir, err)
+    except OSError as err:
+      print('Failed to create log directory "%s": %s' % (log_dir, err), file=sys.stderr)
       raise err
 
   # Remember where our log directory is
@@ -188,7 +195,7 @@ def basic_logging(proc_name, log_dir=None):
 
 def fancy_logging():
   """Configure logging into a buffer for /logs endpoint."""
-  from log_buffer import FixedBufferHandler
+  from .log_buffer import FixedBufferHandler
 
   BUFFER_SIZE = 1500 * 200 # This is the size in characters, not bytes. Targets about 1500 rows.
   buffer_handler = FixedBufferHandler(BUFFER_SIZE)

+ 4 - 2
desktop/core/src/desktop/log/access.py

@@ -19,7 +19,9 @@
 Decorators and methods related to access log.
 This assumes a single-threaded server.
 """
+from __future__ import division
 
+from past.utils import old_div
 import logging
 import re
 import resource
@@ -73,7 +75,7 @@ class AccessInfo(dict):
   """
   def __init__(self, request):
     self['username'] = request.user.username or '-anon-'
-    if request.META.has_key('HTTP_X_FORWARDED_FOR'):
+    if 'HTTP_X_FORWARDED_FOR' in request.META:
       self['remote_ip'] = request.META.get('HTTP_X_FORWARDED_FOR', '-')
     else:
       self['remote_ip'] = request.META.get('REMOTE_ADDR', '-')
@@ -94,7 +96,7 @@ class AccessInfo(dict):
     if sys.platform == 'darwin':
       rusage_denom = rusage_denom * 1024
     # get peak memory usage, bytes on OSX, Kilobytes on Linux
-    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom
+    return old_div(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, rusage_denom)
 
   def log(self, level, msg=None, start_time=None):
     is_instrumentation = desktop.conf.INSTRUMENTATION.get()

+ 1 - 0
desktop/core/src/desktop/log/log_buffer.py

@@ -22,6 +22,7 @@ of log message around for us to view in case of emergency.
 This log handler lets us do that.
 """
 
+from builtins import object
 import logging, collections
 
 class FixedBuffer(object):

+ 2 - 1
desktop/core/src/desktop/log/log_buffer_test.py

@@ -18,7 +18,8 @@
 # A couple of test cases for the log buffer
 #
 
-import log_buffer
+from __future__ import absolute_import
+from desktop.log import log_buffer
 import logging
 import unittest
 

+ 47 - 41
desktop/core/src/desktop/manage_entry.py

@@ -15,6 +15,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+from future import standard_library
+standard_library.install_aliases()
 import logging
 import os
 import os.path
@@ -30,7 +33,7 @@ def _deprecation_check(arg0):
   if os.path.basename(arg0) == 'desktop':
     to_use = os.path.join(os.path.dirname(arg0), 'hue')
     msg = "Warning: '%s' has been deprecated. Please use '%s' instead." % (arg0, to_use)
-    print >> sys.stderr, msg
+    print(msg, file=sys.stderr)
     LOG.warn(msg)
 
 def reload_with_cm_env(cm_managed):
@@ -38,15 +41,15 @@ def reload_with_cm_env(cm_managed):
     from django.db.backends.oracle.base import Oracle_datetime
   except:
     if 'LD_LIBRARY_PATH' in os.environ:
-      print "We need to reload the process to include LD_LIBRARY_PATH for Oracle backend"
+      print("We need to reload the process to include LD_LIBRARY_PATH for Oracle backend")
       try:
         if cm_managed:
           sys.argv.append("--cm-managed")
  
         sys.argv.append("--skip-reload")
         os.execv(sys.argv[0], sys.argv)
-      except Exception, exc:
-        print 'Failed re-exec: %s' % exc
+      except Exception as exc:
+        print('Failed re-exec: %s' % exc)
         sys.exit(1)
 
 def entry():
@@ -60,7 +63,7 @@ def entry():
   os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings')
   cm_config_file = '/etc/cloudera-scm-agent/config.ini'
   ld_path_orig = None
-  if "LD_LIBRARY_PATH" in os.environ.keys():
+  if "LD_LIBRARY_PATH" in list(os.environ.keys()):
     ld_path_orig = os.environ["LD_LIBRARY_PATH"]
 
   # What's the subcommand being run?
@@ -89,17 +92,20 @@ def entry():
     prof_id = subcommand = sys.argv[1]
     #Check if this is a CM managed cluster
     if os.path.isfile(cm_config_file) and not cm_managed and not skip_reload:
-        print "ALERT: This appears to be a CM Managed environment"
-        print "ALERT: HUE_CONF_DIR must be set when running hue commands in CM Managed environment"
-        print "ALERT: Please run 'hue <command> --cm-managed'"
+        print("ALERT: This appears to be a CM Managed environment")
+        print("ALERT: HUE_CONF_DIR must be set when running hue commands in CM Managed environment")
+        print("ALERT: Please run 'hue <command> --cm-managed'")
   else:
     prof_id = str(os.getpid())
 
   # CM managed configure env vars
   if cm_managed:
-    import ConfigParser
-    from ConfigParser import NoOptionError
-    config = ConfigParser.RawConfigParser()
+    if sys.version_info[0] > 2:
+      from configparser import NoOptionError, RawConfigParser
+    else:
+      from ConfigParser import NoOptionError, RawConfigParser
+
+    config = RawConfigParser()
     config.read(cm_config_file)
     try:
       cm_agent_run_dir = config.get('General', 'agent_wide_credential_cache_location')
@@ -136,19 +142,19 @@ def entry():
             empty, hue_conf_dir = line.split("directory=")
             os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip()
     else:
-      print "This appears to be a CM managed cluster, but the"
-      print "supervisor/include file for Hue could not be found"
-      print "in order to successfully run commands that access"
-      print "the database you need to set the following env vars:"
-      print ""
-      print "  export JAVA_HOME=<java_home>"
-      print "  export HUE_CONF_DIR=\"%s/`ls -1 %s | grep %s | sort -n | tail -1 `\"" % (cm_processs_dir, cm_process_dir, cm_hue_string)
-      print "  export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1"
-      print "  export HUE_DATABASE_PASSWORD=<hueDBpassword>"
-      print "If using Oracle as your database:"
-      print "  export LD_LIBRARY_PATH=/path/to/instantclient"
-      print ""
-      print "If the above does not work, make sure Hue has been started on this server."
+      print("This appears to be a CM managed cluster, but the")
+      print("supervisor/include file for Hue could not be found")
+      print("in order to successfully run commands that access")
+      print("the database you need to set the following env vars:")
+      print("")
+      print("  export JAVA_HOME=<java_home>")
+      print("  export HUE_CONF_DIR=\"%s/`ls -1 %s | grep %s | sort -n | tail -1 `\"" % (cm_processs_dir, cm_process_dir, cm_hue_string))
+      print("  export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1")
+      print("  export HUE_DATABASE_PASSWORD=<hueDBpassword>")
+      print("If using Oracle as your database:")
+      print("  export LD_LIBRARY_PATH=/path/to/instantclient")
+      print("")
+      print("If the above does not work, make sure Hue has been started on this server.")
 
     if not envline == None:
       empty, environment = envline.split("environment=")
@@ -160,7 +166,7 @@ def entry():
           os.environ[envkey] = envval
 
     #Set JAVA_HOME
-    if "JAVA_HOME" not in os.environ.keys():
+    if "JAVA_HOME" not in list(os.environ.keys()):
       if os.path.isfile('/usr/lib64/cmf/service/common/cloudera-config.sh'):
         locate_java = subprocess.Popen(
           ['bash', '-c', '. /usr/lib64/cmf/service/common/cloudera-config.sh; locate_java_home'], stdout=subprocess.PIPE,
@@ -182,14 +188,14 @@ def entry():
       if JAVA_HOME != "UNKNOWN":
         os.environ["JAVA_HOME"] = JAVA_HOME
 
-      if "JAVA_HOME" not in os.environ.keys():
-        print "JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable"
-        print "  export JAVA_HOME=<java_home>"
+      if "JAVA_HOME" not in list(os.environ.keys()):
+        print("JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable")
+        print("  export JAVA_HOME=<java_home>")
         sys.exit(1)
 
     #Make sure we set Oracle Client if configured
-    if "LD_LIBRARY_PATH" not in os.environ.keys():
-      if "SCM_DEFINES_SCRIPTS" in os.environ.keys():
+    if "LD_LIBRARY_PATH" not in list(os.environ.keys()):
+      if "SCM_DEFINES_SCRIPTS" in list(os.environ.keys()):
         for scm_script in os.environ["SCM_DEFINES_SCRIPTS"].split(":"):
           if "ORACLE" in scm_script:
             if os.path.isfile(scm_script):
@@ -199,12 +205,12 @@ def entry():
                   var, oracle_ld_path = line.split("=")
                   os.environ["LD_LIBRARY_PATH"] = oracle_ld_path
 
-    if "LD_LIBRARY_PATH" not in os.environ.keys():
-      print "LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database"
-      print "then it must be set, if not, you can ignore"
-      print "  export LD_LIBRARY_PATH=/path/to/instantclient"
+    if "LD_LIBRARY_PATH" not in list(os.environ.keys()):
+      print("LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database")
+      print("then it must be set, if not, you can ignore")
+      print("  export LD_LIBRARY_PATH=/path/to/instantclient")
 
-  if "LD_LIBRARY_PATH" in os.environ.keys():
+  if "LD_LIBRARY_PATH" in list(os.environ.keys()):
     if ld_path_orig is not None and ld_path_orig == os.environ["LD_LIBRARY_PATH"]:
       skip_reload = True
 
@@ -217,16 +223,16 @@ def entry():
       _profile(prof_id, lambda: execute_from_command_line(sys.argv))
     else:
       execute_from_command_line(sys.argv)
-  except ImproperlyConfigured, e:
+  except ImproperlyConfigured as e:
     if len(sys.argv) > 1 and sys.argv[1] == 'is_db_alive' and 'oracle' in str(e).lower():
-      print >> sys.stderr, e # Oracle connector is improperly configured
+      print(e, file=sys.stderr) # Oracle connector is improperly configured
       sys.exit(10)
     else:
       raise e
-  except subprocess.CalledProcessError, e:
+  except subprocess.CalledProcessError as e:
     if "altscript.sh" in str(e).lower():
-      print "%s" % e
-      print "HUE_CONF_DIR seems to be set to CM location and '--cm-managed' flag not used"
+      print("%s" % e)
+      print("HUE_CONF_DIR seems to be set to CM location and '--cm-managed' flag not used")
 
 def _profile(prof_id, func):
   """
@@ -250,4 +256,4 @@ def _profile(prof_id, func):
     prof.dump_stats(PROF_DAT)
     # Sort the calls by time spent and show top 50
     pstats.Stats(PROF_DAT).sort_stats('time').print_stats(50)
-    print >>sys.stderr, "Complete profile data in %s" % (PROF_DAT,)
+    print("Complete profile data in %s" % (PROF_DAT,), file=sys.stderr)

+ 10 - 9
desktop/core/src/desktop/management/commands/config_dump.py

@@ -21,6 +21,7 @@ Note that this dump representation is not machine readable;
 this command is still a few steps away from dumping a ConfigObj-compatible,
 textual representation.
 """
+from __future__ import print_function
 from django.core.management.base import BaseCommand
 import desktop.appmanager
 import textwrap
@@ -36,16 +37,16 @@ class Command(BaseCommand):
 
   """Prints documentation for configuration."""
   def handle(self, *args, **options):
-    print _("Dumping configuration...")
-    print
+    print(_("Dumping configuration..."))
+    print()
     self.recurse(desktop.lib.conf.GLOBAL_CONFIG)
 
   def p(self, s):
-    print " "*self.indent + s
+    print(" "*self.indent + s)
 
   def fill(self, s):
-    print textwrap.fill(s.strip(),
-      initial_indent=" "*self.indent, subsequent_indent=" "*self.indent)
+    print(textwrap.fill(s.strip(),
+      initial_indent=" "*self.indent, subsequent_indent=" "*self.indent))
     
 
   def recurse(self, config_obj):
@@ -57,10 +58,10 @@ class Command(BaseCommand):
 
       self.p("%s:" % key)
       self.indent += 2
-      print textwrap.fill(config_obj.config.help or _("No help available."),
-        initial_indent=" "*self.indent, subsequent_indent=" "*self.indent)
-      print
-      for v in config_obj.get().values():
+      print(textwrap.fill(config_obj.config.help or _("No help available."),
+        initial_indent=" "*self.indent, subsequent_indent=" "*self.indent))
+      print()
+      for v in list(config_obj.get().values()):
         self.recurse(v)
       self.indent -= 2
 

+ 5 - 4
desktop/core/src/desktop/management/commands/convert_documents.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import logging
 import sys
 import time
@@ -28,7 +29,7 @@ from desktop.converters import DocumentConverter
 class Command(BaseCommand):
 
   def handle(self, *args, **options):
-    print 'Starting document conversions...\n'
+    print('Starting document conversions...\n')
     try:
       with transaction.atomic():
         users = User.objects.all()
@@ -42,8 +43,8 @@ class Command(BaseCommand):
           logging.info("Document conversions for user:%s took %.3f seconds" % (user.username, time.time() - start_time))
 
           if converter.failed_doc_ids:
-            print >> sys.stderr, 'Failed to import %d document(s) for user: %s - %s' % (len(converter.failed_doc_ids), user.username, converter.failed_doc_ids)
-    except Exception, e:
+            print('Failed to import %d document(s) for user: %s - %s' % (len(converter.failed_doc_ids), user.username, converter.failed_doc_ids), file=sys.stderr)
+    except Exception as e:
       logging.exception("Failed to execute the document conversions.")
 
-    print 'Finished running document conversions.\n'
+    print('Finished running document conversions.\n')

+ 1 - 1
desktop/core/src/desktop/management/commands/create_desktop_app.py

@@ -55,7 +55,7 @@ class Command(BaseCommand):
       raise CommandError(_("%(name)r is not a valid application name. Use only numbers, letters and underscores.") % {'name': app_name})
     try:
       os.makedirs(app_dir)
-    except OSError, e:
+    except OSError as e:
       raise CommandError(e)
 
     copy_template(app_template, app_dir, app_name)

+ 1 - 1
desktop/core/src/desktop/management/commands/create_proxy_app.py

@@ -57,7 +57,7 @@ class Command(BaseCommand):
       raise CommandError(_("%(name)r is not a valid application name. Use only numbers, letters and underscores.") % {'name': app_name})
     try:
       os.makedirs(app_dir)
-    except OSError, e:
+    except OSError as e:
       raise CommandError(e)
 
     copy_template(app_template, app_dir, app_name, app_url)

+ 3 - 1
desktop/core/src/desktop/management/commands/create_test_fs.py

@@ -13,6 +13,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from __future__ import print_function
 import os
 
 from django.core.management.base import BaseCommand
@@ -28,4 +30,4 @@ class Command(BaseCommand):
     if not os.path.isdir(fs_dir):
       os.makedirs(fs_dir)
     fs_for_testing.create(fs_dir)
-    print _("Created fs in: %(dir)s") % {'dir': fs_dir}
+    print(_("Created fs in: %(dir)s") % {'dir': fs_dir})

+ 2 - 2
desktop/core/src/desktop/management/commands/create_user_directories.py

@@ -48,7 +48,7 @@ class Command(BaseCommand):
       try:
         user = User.objects.get(username=options['username'])
         users = [user]
-      except Exception, e:
+      except Exception as e:
         msg = 'Failed to get user with username %s: %s' % (options['username'], str(e))
         self.stdout.write(msg)
         LOG.exception(msg)
@@ -59,7 +59,7 @@ class Command(BaseCommand):
         self.stdout.write(msg)
         LOG.debug(msg)
         Document2.objects.create_user_directories(user)
-      except Exception, e:
+      except Exception as e:
         msg = 'Failed to create user directories for user %s: %s' % (user.username, str(e))
         self.stdout.write(msg)
         LOG.warn(msg)

+ 2 - 2
desktop/core/src/desktop/management/commands/desktop_document_cleanup.py

@@ -53,7 +53,7 @@ class Command(BaseCommand):
                     default=30),
         )
 
-    except AttributeError, e:
+    except AttributeError as e:
         baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
         if baseoption_test:
             def add_arguments(self, parser):
@@ -88,7 +88,7 @@ class Command(BaseCommand):
             try:
                 objClass.objects.filter(pk__in=list(deleteObjects)).delete()
                 errorCount = 0
-            except DatabaseError, e:
+            except DatabaseError as e:
                 LOG.info("Non Fatal Exception: %s: %s" % (e.__class__.__name__, e))
                 errorCount += 1
                 if errorCount > 9 and deleteRecords == 1:

+ 3 - 1
desktop/core/src/desktop/management/commands/generate_mdl.py

@@ -17,7 +17,9 @@
 """
 Dumps a Cloudera Manager Monitor Descriptor file.
 """
+from __future__ import print_function
 
+from past.builtins import cmp
 import json
 
 from django.core.management.base import BaseCommand
@@ -52,4 +54,4 @@ class Command(BaseCommand):
         ],
     }
 
-    print json.dumps(d)
+    print(json.dumps(d))

+ 3 - 3
desktop/core/src/desktop/management/commands/is_db_alive.py

@@ -71,10 +71,10 @@ class Command(BaseCommand):
     try:
       db_conn = connections['default']
       db_conn.cursor()
-    except ImproperlyConfigured, e:
+    except ImproperlyConfigured as e:
       self.stderr.write('DB improperly configured: %s' % e)
       sys.exit(10)
-    except OperationalError, e:
+    except OperationalError as e:
       self.stderr.write('Error accessing DB: %s' % e)
       error = str(e)
 
@@ -89,7 +89,7 @@ class Command(BaseCommand):
         sys.exit(8)
       else: # Any connection error that we can't make sense of
         sys.exit(4)
-    except Exception, e:
+    except Exception as e:
       self.stderr.write('Error accessing DB: %s' % e)
       error = str(e)
 

+ 1 - 0
desktop/core/src/desktop/management/commands/run_windmill.py

@@ -18,6 +18,7 @@
 Django management command that invokes windmill tests,
 after appropriate setup.
 """
+from builtins import object
 import sys
 import time
 from optparse import make_option

+ 3 - 2
desktop/core/src/desktop/management/commands/runcherrypyserver.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 # a thirdparty project
 
+from __future__ import print_function
 import logging
 import pprint
 import sys
@@ -114,11 +115,11 @@ def runcpserver(argset=[], **kwargs):
         options[k.lower()] = v
     
     if "help" in options:
-        print CPSERVER_HELP
+        print(CPSERVER_HELP)
         return
 
     # Start the webserver
-    print _('starting server with options:')
+    print(_('starting server with options:'))
     pprint.pprint(options)
 
     start_server(options)

+ 5 - 3
desktop/core/src/desktop/management/commands/test.py

@@ -20,6 +20,8 @@ Desktop-aware test runner.
 Django's "test" command merely executes the test_runner,
 so we circumvent it entirely and create our own.
 """
+from __future__ import print_function
+from builtins import object
 from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.test.utils import get_runner
@@ -134,7 +136,7 @@ class Command(BaseCommand):
     #south_logger.setLevel(logging.INFO)
 
     if len(args) == 0:
-      print self.help
+      print(self.help)
       sys.exit(1)
 
     nose_args = None
@@ -153,10 +155,10 @@ class Command(BaseCommand):
     elif args[0] in ("specific", "nose"):
       nose_args = args
     elif args[0] == "list_modules":
-      print '\n'.join(all_apps)
+      print('\n'.join(all_apps))
       sys.exit(0)
     else:
-      print self.help
+      print(self.help)
       sys.exit(1)
 
     if nose_args:

+ 3 - 2
desktop/core/src/desktop/management/commands/version.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import os
 
 from django.conf import settings
@@ -28,7 +29,7 @@ class Command(BaseCommand):
   def handle(self, *args, **options):
     cdh_path = os.path.join(get_desktop_root(), '..', 'cloudera', 'cdh_version.properties')
     if os.path.exists(cdh_path):
-      print open(cdh_path).read()
+      print(open(cdh_path).read())
     else:
-      print settings.HUE_DESKTOP_VERSION
+      print(settings.HUE_DESKTOP_VERSION)
 

+ 4 - 1
desktop/core/src/desktop/metrics.py

@@ -16,6 +16,9 @@
 
 from __future__ import absolute_import
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
 import gc
 import logging
 import multiprocessing
@@ -65,7 +68,7 @@ global_registry().gauge_callback(
 
 # ------------------------------------------------------------------------------
 
-for i in xrange(3):
+for i in range(3):
   global_registry().gauge_callback(
       name='python.gc.generation.%s' % i,
       callback=lambda: gc.get_count()[i],

+ 12 - 12
desktop/core/src/desktop/middleware.py

@@ -17,6 +17,7 @@
 
 from __future__ import absolute_import
 
+from builtins import object
 import inspect
 import json
 import logging
@@ -235,16 +236,16 @@ class AppSpecificMiddleware(object):
       try:
           dot = middleware_path.rindex('.')
       except ValueError:
-          raise exceptions.ImproperlyConfigured, _('%(module)s isn\'t a middleware module.') % {'module': middleware_path}
+          raise exceptions.ImproperlyConfigured(_('%(module)s isn\'t a middleware module.') % {'module': middleware_path})
       mw_module, mw_classname = middleware_path[:dot], middleware_path[dot+1:]
       try:
           mod = __import__(mw_module, {}, {}, [''])
-      except ImportError, e:
-          raise exceptions.ImproperlyConfigured, _('Error importing middleware %(module)s: "%(error)s".') % {'module': mw_module, 'error': e}
+      except ImportError as e:
+          raise exceptions.ImproperlyConfigured(_('Error importing middleware %(module)s: "%(error)s".') % {'module': mw_module, 'error': e})
       try:
           mw_class = getattr(mod, mw_classname)
       except AttributeError:
-          raise exceptions.ImproperlyConfigured, _('Middleware module "%(module)s" does not define a "%(class)s" class.') % {'module': mw_module, 'class':mw_classname}
+          raise exceptions.ImproperlyConfigured(_('Middleware module "%(module)s" does not define a "%(class)s" class.') % {'module': mw_module, 'class':mw_classname})
 
       try:
         mw_instance = mw_class()
@@ -255,9 +256,8 @@ class AppSpecificMiddleware(object):
       # We need to make sure we don't have a process_request function because we don't know what
       # application will handle the request at the point process_request is called
       if hasattr(mw_instance, 'process_request'):
-        raise exceptions.ImproperlyConfigured, \
-              _('AppSpecificMiddleware module "%(module)s" has a process_request function' + \
-              ' which is impossible.') % {'module': middleware_path}
+        raise exceptions.ImproperlyConfigured(_('AppSpecificMiddleware module "%(module)s" has a process_request function' + \
+              ' which is impossible.') % {'module': middleware_path})
       if hasattr(mw_instance, 'process_view'):
         result['view'].append(mw_instance.process_view)
       if hasattr(mw_instance, 'process_response'):
@@ -311,7 +311,7 @@ class LoginAndPermissionMiddleware(object):
       # Until we get Django 1.3 and resolve returning the URL name, we just do a match of the name of the view
       try:
         access_view = 'access_view:%s:%s' % (request._desktop_app, resolve(request.path)[0].__name__)
-      except Exception, e:
+      except Exception as e:
         access_log(request, 'error checking view perm: %s' % e, level=access_log_level)
         access_view = ''
 
@@ -387,7 +387,7 @@ class AuditLoggingMiddleware(object):
       if hasattr(request, 'audit') and request.audit is not None:
         self._log_message(request, response)
         response['audited'] = True
-    except Exception, e:
+    except Exception as e:
       LOG.error('Could not audit the request: %s' % e)
     return response
 
@@ -433,7 +433,7 @@ class AuditLoggingMiddleware(object):
 try:
   import tidylib
   _has_tidylib = True
-except Exception, ex:
+except Exception as ex:
   # The exception type is not ImportError. It's actually an OSError.
   logging.warn("Failed to import tidylib (for debugging). Is libtidy installed?")
   _has_tidylib = False
@@ -462,8 +462,8 @@ class HtmlValidationMiddleware(object):
     try:
       self._outdir = os.path.join(tempfile.gettempdir(), 'hue_html_validation')
       if not os.path.isdir(self._outdir):
-        os.mkdir(self._outdir, 0755)
-    except Exception, ex:
+        os.mkdir(self._outdir, 0o755)
+    except Exception as ex:
       self._logger.exception('Failed to get temp directory: %s', (ex,))
       self._outdir = tempfile.mkdtemp(prefix='hue_html_validation-')
 

+ 1 - 1
desktop/core/src/desktop/middleware_test.py

@@ -90,7 +90,7 @@ def test_audit_logging_middleware_enable():
       audit = open(log_path).readlines()
       for line in audit:
         audit_json = json.loads(line)
-        audit_record = audit_json.values()[0]
+        audit_record = list(audit_json.values())[0]
         assert_equal('test_audit_logging', audit_record['user'], audit_record)
         assert_equal('/useradmin/permissions/edit/beeswax/access', audit_record['url'], audit_record)
 

+ 43 - 32
desktop/core/src/desktop/models.py

@@ -15,11 +15,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import next
+from builtins import object
 import calendar
 import json
 import logging
 import os
-import urllib
+import sys
 import uuid
 
 from collections import OrderedDict
@@ -36,7 +41,7 @@ from django.db.models import Q
 from django.db.models.query import QuerySet
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 
-from settings import HUE_DESKTOP_VERSION
+from desktop.settings import HUE_DESKTOP_VERSION
 
 from dashboard.conf import get_engines, HAS_REPORT_ENABLED
 from kafka.conf import has_kafka
@@ -52,6 +57,12 @@ from desktop.redaction import global_redaction_engine
 from desktop.settings import DOCUMENT2_SEARCH_MAX_LENGTH
 from desktop.auth.backend import is_admin
 
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+  from urllib.parse import quote as urllib_quote
+else:
+  from urllib import quote as urllib_quote
+
 LOG = logging.getLogger(__name__)
 
 SAMPLE_USER_ID = 1100713
@@ -90,7 +101,7 @@ def _version_from_properties(f):
 PREFERENCE_IS_WELCOME_TOUR_SEEN = 'is_welcome_tour_seen'
 
 class HueUser(auth_models.User):
-  class Meta:
+  class Meta(object):
     proxy = True
 
   def __init__(self, *args, **kwargs):
@@ -158,7 +169,7 @@ class DefaultConfiguration(models.Model):
 
   objects = DefaultConfigurationManager()
 
-  class Meta:
+  class Meta(object):
     ordering = ["app", "-is_default", "user"]
 
 
@@ -291,7 +302,7 @@ class DocumentTag(models.Model):
 
   objects = DocumentTagManager()
 
-  class Meta:
+  class Meta(object):
     unique_together = ('owner', 'tag')
 
 
@@ -433,7 +444,7 @@ class DocumentManager(models.Manager):
               if not job.managed:
                 doc.extra = 'jobsub'
                 doc.save()
-    except Exception, e:
+    except Exception as e:
       LOG.exception('error syncing oozie')
 
     try:
@@ -445,7 +456,7 @@ class DocumentManager(models.Manager):
             doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.desc, extra=job.type)
             if job.is_trashed:
               doc.send_to_trash()
-    except Exception, e:
+    except Exception as e:
       LOG.exception('error syncing beeswax')
 
     try:
@@ -455,7 +466,7 @@ class DocumentManager(models.Manager):
         with transaction.atomic():
           for job in find_jobs_with_no_doc(PigScript):
             Document.objects.link(job, owner=job.owner, name=job.dict['name'], description='')
-    except Exception, e:
+    except Exception as e:
       LOG.exception('error syncing pig')
 
     try:
@@ -478,7 +489,7 @@ class DocumentManager(models.Manager):
                 dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard', owner=owner, description=dashboard.label, data=dashboard.properties)
                 Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label, extra='search-dashboard')
                 dashboard.save()
-    except Exception, e:
+    except Exception as e:
       LOG.exception('error syncing search')
 
     try:
@@ -498,7 +509,7 @@ class DocumentManager(models.Manager):
             else:
               extra = ''
             doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description, extra=extra)
-    except Exception, e:
+    except Exception as e:
       LOG.exception('error syncing Document2')
 
 
@@ -508,7 +519,7 @@ class DocumentManager(models.Manager):
         for doc in Document.objects.filter(tags=None):
           default_tag = DocumentTag.objects.get_default_tag(doc.owner)
           doc.tags.add(default_tag)
-      except Exception, e:
+      except Exception as e:
         LOG.exception('error adding at least one tag to docs')
 
       # Make sure all the sample user documents are shared.
@@ -523,7 +534,7 @@ class DocumentManager(models.Manager):
 
             doc.save()
             Document.objects.filter(id=doc.id).update(last_modified=doc_last_modified)
-      except Exception, e:
+      except Exception as e:
         LOG.exception('error sharing sample user documents')
 
       # For now remove the default tag from the examples
@@ -531,7 +542,7 @@ class DocumentManager(models.Manager):
         for doc in Document.objects.filter(tags__tag=DocumentTag.EXAMPLE):
           default_tag = DocumentTag.objects.get_default_tag(doc.owner)
           doc.tags.remove(default_tag)
-      except Exception, e:
+      except Exception as e:
         LOG.exception('error removing default tags')
 
       # ------------------------------------------------------------------------
@@ -600,7 +611,7 @@ class DocumentManager(models.Manager):
           if docs.exists():
             LOG.info('Deleting %s documents' % docs.count())
             docs.delete()
-      except Exception, e:
+      except Exception as e:
         LOG.exception('Error in sync while attempting to delete documents with no object: %s' % e)
 
 
@@ -622,7 +633,7 @@ class Document(models.Model):
 
   objects = DocumentManager()
 
-  class Meta:
+  class Meta(object):
     unique_together = ('content_type', 'object_id')
 
   def __unicode__(self):
@@ -750,7 +761,7 @@ class Document(models.Model):
         return staticfiles_storage.url(apps[self.content_type.app_label].icon_path)
       else:
         return staticfiles_storage.url('desktop/art/icon_hue_48.png')
-    except Exception, e:
+    except Exception as e:
       LOG.warn(force_unicode(e))
       return staticfiles_storage.url('desktop/art/icon_hue_48.png')
 
@@ -766,7 +777,7 @@ class Document(models.Model):
 
     Example of input: {'read': {'user_ids': [1, 2, 3], 'group_ids': [1, 2, 3]}}
     """
-    for name, perm in perms_dict.iteritems():
+    for name, perm in perms_dict.items():
       users = groups = None
       if perm.get('user_ids'):
         users = auth_models.User.objects.in_bulk(perm.get('user_ids'))
@@ -880,7 +891,7 @@ class DocumentPermission(models.Model):
 
   objects = DocumentPermissionManager()
 
-  class Meta:
+  class Meta(object):
     unique_together = ('doc', 'perms')
 
 
@@ -1090,7 +1101,7 @@ class Document2(models.Model):
 
   objects = Document2Manager()
 
-  class Meta:
+  class Meta(object):
     unique_together = ('uuid', 'version', 'is_history')
     ordering = ["-last_modified", "name"]
 
@@ -1107,7 +1118,7 @@ class Document2(models.Model):
 
   @property
   def path(self):
-    quoted_name = urllib.quote(self.name.encode('utf-8'))
+    quoted_name = urllib_quote(self.name.encode('utf-8'))
     if self.parent_directory:
       return '%s/%s' % (self.parent_directory.path, quoted_name)
     else:
@@ -1341,7 +1352,7 @@ class Document2(models.Model):
         # For directories, update all children recursively with same permissions
         for child in self.children.all():
           child.share(user, name, users, groups)
-    except Exception, e:
+    except Exception as e:
       raise PopupException(_("Failed to share document: %s") % e)
     return self
 
@@ -1373,7 +1384,7 @@ class Document2(models.Model):
 
     try:
       doc = self.doc.get()
-    except Exception, e:
+    except Exception as e:
       LOG.error('Exception when retrieving document object for saved query: %s' % e)
       doc = Document.objects.link(
         self,
@@ -1465,7 +1476,7 @@ class Directory(Document2):
 
   objects = DirectoryManager()
 
-  class Meta:
+  class Meta(object):
     proxy = True
 
   def get_children_documents(self):
@@ -1532,7 +1543,7 @@ class Document2Permission(models.Model):
   # link = models.CharField(default=uuid_default, max_length=255, unique=True) # Short link like dropbox
   # embed
 
-  class Meta:
+  class Meta(object):
     unique_together = ('doc', 'perms')
 
   def to_dict(self):
@@ -1558,7 +1569,7 @@ def get_cluster_config(user):
 ANALYTIC_DB = 'altus'
 
 
-class ClusterConfig():
+class ClusterConfig(object):
   """
   Configuration of the apps and engines that each individual user sees on the core Hue.
   Fine grained Hue permissions and available apps are leveraged here in order to render the correct UI.
@@ -1620,7 +1631,7 @@ class ClusterConfig():
     if not apps:
       raise PopupException(_('No permission to any app.'))
 
-    default_app = apps.values()[0]
+    default_app = list(apps.values())[0]
     default_interpreter = default_app.get('interpreters')
 
     try:
@@ -1767,7 +1778,7 @@ class ClusterConfig():
         'displayName': _('Files'),
         'buttonName': _('Browse'),
         'tooltip': _('Files'),
-        'page': '/filebrowser/' + (not self.user.is_anonymous() and 'view=' + urllib.quote(self.user.get_home_directory().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS) or '')
+        'page': '/filebrowser/' + (not self.user.is_anonymous() and 'view=' + urllib_quote(self.user.get_home_directory().encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS) or '')
       })
 
     if 'filebrowser' in self.apps and not IS_EMBEDDED.get() and fsmanager.is_enabled_and_has_access('s3a', self.user):
@@ -1776,7 +1787,7 @@ class ClusterConfig():
         'displayName': _('S3'),
         'buttonName': _('Browse'),
         'tooltip': _('S3'),
-        'page': '/filebrowser/view=' + urllib.quote('S3A://'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+        'page': '/filebrowser/view=' + urllib_quote('S3A://'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
       })
 
     if 'filebrowser' in self.apps and ANALYTIC_DB not in self.cluster_type and fsmanager.is_enabled_and_has_access('adl', self.user):
@@ -1785,7 +1796,7 @@ class ClusterConfig():
         'displayName': _('ADLS'),
         'buttonName': _('Browse'),
         'tooltip': _('ADLS'),
-        'page': '/filebrowser/view=' + urllib.quote('adl:/'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+        'page': '/filebrowser/view=' + urllib_quote('adl:/'.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
       })
       
     if 'filebrowser' in self.apps and ANALYTIC_DB not in self.cluster_type and fsmanager.is_enabled_and_has_access('abfs', self.user):
@@ -1949,7 +1960,7 @@ class ClusterConfig():
   def get_hive_metastore_interpreters(self):
     return [interpreter['type'] for interpreter in get_ordered_interpreters(self.user) if interpreter == 'hive' or interpreter == 'hms']
 
-class Cluster():
+class Cluster(object):
 
   def __init__(self, user):
     self.user = user
@@ -1958,7 +1969,7 @@ class Cluster():
     if IS_MULTICLUSTER_ONLY.get():
       self.data = self.clusters['Altus'] # Backward compatibility
     else:
-      self.data = self.clusters.values()[0] # Next: CLUSTER_ID.get() or user persisted
+      self.data = list(self.clusters.values())[0] # Next: CLUSTER_ID.get() or user persisted
 
   def get_type(self):
     return self.data['type']
@@ -1975,7 +1986,7 @@ def _get_apps(user, section=None):
   other_apps = []
   if user.is_authenticated():
     apps_list = appmanager.get_apps_dict(user)
-    apps = apps_list.values()
+    apps = list(apps_list.values())
     for app in apps:
       if app.display_name not in [
           'beeswax', 'hive', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser',

+ 2 - 2
desktop/core/src/desktop/old_migrations/0013_auto__add_unique_documenttag_owner_tag.py

@@ -39,7 +39,7 @@ class Migration(SchemaMigration):
 
                 DocumentTag.objects.filter(id__in=docs).delete()
             db.commit_transaction()
-        except Exception, e:
+        except Exception as e:
             db.rollback_transaction()
             raise e
 
@@ -48,7 +48,7 @@ class Migration(SchemaMigration):
             # Adding unique constraint on 'DocumentTag', fields ['owner', 'tag']
             db.create_unique(u'desktop_documenttag', ['owner_id', 'tag'])
             db.commit_transaction()
-        except Exception, e:
+        except Exception as e:
             db.rollback_transaction()
             raise e
 

+ 2 - 2
desktop/core/src/desktop/old_migrations/0014_auto__add_unique_document_content_type_object_id.py

@@ -39,7 +39,7 @@ class Migration(SchemaMigration):
 
                 Document.objects.filter(id__in=docs).delete()
             db.commit_transaction()
-        except Exception, e:
+        except Exception as e:
             db.rollback_transaction()
             raise e
 
@@ -48,7 +48,7 @@ class Migration(SchemaMigration):
             # Adding unique constraint on 'Document', fields ['content_type', 'object_id']
             db.create_unique(u'desktop_document', ['content_type_id', 'object_id'])
             db.commit_transaction()
-        except Exception, e:
+        except Exception as e:
             db.rollback_transaction()
             raise e
 

+ 2 - 2
desktop/core/src/desktop/old_migrations/0015_auto__add_unique_documentpermission_doc_perms.py

@@ -39,7 +39,7 @@ class Migration(SchemaMigration):
 
                 DocumentPermission.objects.filter(id__in=docs).delete()
             db.commit_transaction()
-        except Exception, e:
+        except Exception as e:
             db.rollback_transaction()
             raise e
 
@@ -48,7 +48,7 @@ class Migration(SchemaMigration):
             # Adding unique constraint on 'DocumentPermission', fields ['doc', 'perms']
             db.create_unique(u'desktop_documentpermission', ['doc_id', 'perms'])
             db.commit_transaction()
-        except Exception, e:
+        except Exception as e:
             db.rollback_transaction()
             raise e
 

+ 1 - 1
desktop/core/src/desktop/old_migrations/0016_auto__add_unique_document2_uuid_version_is_history.py

@@ -58,7 +58,7 @@ class Migration(SchemaMigration):
             # Adding unique constraint on 'Document2', fields ['uuid', 'version', 'is_history']
             db.create_unique(u'desktop_document2', ['uuid', 'version', 'is_history'])
             db.commit_transaction()
-        except Exception, e:
+        except Exception as e:
             db.rollback_transaction()
             raise e
 

+ 3 - 2
desktop/core/src/desktop/redaction/engine.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import json
 import re
 
@@ -163,7 +164,7 @@ def parse_redaction_policy_from_file(filename):
     rules = [parse_one_rule_from_dict(rule) for rule in rules]
 
     if scheme:
-      raise ValueError('Redaction policy contains unknown field(s): %s' % scheme.keys())
+      raise ValueError('Redaction policy contains unknown field(s): %s' % list(scheme.keys()))
 
     return RedactionPolicy(rules)
 
@@ -204,7 +205,7 @@ def parse_one_rule_from_dict(rule):
     raise ValueError('Redaction rule is missing `replace` field')
 
   if rule:
-    raise ValueError('Redaction rule contains unknown field(s): %s' % rule.keys())
+    raise ValueError('Redaction rule contains unknown field(s): %s' % list(rule.keys()))
 
   return RedactionRule(trigger, search, replace, case_sensitive)
 

+ 8 - 3
desktop/core/src/desktop/redaction/tests.py

@@ -15,6 +15,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import zip
+from builtins import range
+from builtins import object
 import json
 import logging
 import os
@@ -395,8 +400,8 @@ class TestRedactionLogFilter(object):
 
     class TestThread(threading.Thread):
       def run(self):
-        for i in xrange(500):
-          message = u''.join(random_utf8_char() for _ in xrange(128))
+        for i in range(500):
+          message = u''.join(random_utf8_char() for _ in range(128))
           redacted_message = policy.redact(message)
 
           if regex.search(redacted_message):
@@ -405,7 +410,7 @@ class TestRedactionLogFilter(object):
               break
 
     threads = []
-    for i in xrange(10):
+    for i in range(10):
       threads.append(TestThread())
 
     for thread in threads:

+ 5 - 3
desktop/core/src/desktop/settings.py

@@ -20,6 +20,8 @@
 # Local customizations are done by symlinking a file
 # as local_settings.py.
 
+from builtins import map
+from builtins import zip
 import gc
 import json
 import logging
@@ -348,9 +350,9 @@ EMAIL_SUBJECT_PREFIX = 'Hue %s - ' % desktop.conf.CLUSTER_ID.get()
 if os.getenv('DESKTOP_DB_CONFIG'):
   conn_string = os.getenv('DESKTOP_DB_CONFIG')
   logging.debug("DESKTOP_DB_CONFIG SET: %s" % (conn_string))
-  default_db = dict(zip(
+  default_db = dict(list(zip(
     ["ENGINE", "NAME", "TEST_NAME", "USER", "PASSWORD", "HOST", "PORT"],
-    conn_string.split(':')))
+    conn_string.split(':'))))
   default_db['NAME'] = default_db['NAME'].replace('#', ':') # For is_db_alive command
 else:
   test_name = os.environ.get('DESKTOP_DB_TEST_NAME', get_desktop_root('desktop-test.db'))
@@ -707,5 +709,5 @@ if desktop.conf.ENABLE_PROMETHEUS.get():
 
   if 'mysql' in DATABASES['default']['ENGINE']:
     DATABASES['default']['ENGINE'] = DATABASES['default']['ENGINE'].replace('django.db.backends', 'django_prometheus.db.backends')
-  for name, val in CACHES.items():
+  for name, val in list(CACHES.items()):
     val['BACKEND'] = val['BACKEND'].replace('django.core.cache.backends', 'django_prometheus.cache.backends')

+ 29 - 22
desktop/core/src/desktop/supervisor.py

@@ -29,7 +29,10 @@ In order to have your application managed by supervisor, you need to add
 an entry_point to your application's egg with the name 'desktop.supervisor.specs'.
 This entry point should point to a SuperviseeSpec instance in your module.
 """
-from daemon.pidlockfile import PIDLockFile
+
+from __future__ import print_function
+from builtins import range
+from builtins import object
 import daemon
 import exceptions
 import grp
@@ -54,6 +57,22 @@ try:
 except AttributeError:
   MyBaseException = exceptions.Exception
 
+if sys.version_info[0] > 2:
+  from daemon.pidfile import TimeoutPIDLockFile
+else:
+  from daemon.pidlockfile import PIDLockFile
+
+  class TimeOutPIDLockFile(PIDLockFile):
+    """A PIDLockFile subclass that passes through a timeout on acquisition."""
+
+    def __init__(self, lockfile, timeout, **kwargs):
+      PIDLockFile.__init__(self, lockfile, **kwargs)
+      self.timeout = timeout
+
+    def __enter__(self):
+      super(TimeOutPIDLockFile, self).acquire(timeout=self.timeout)
+      return self
+
 PROC_NAME = 'supervisor'
 LOG = logging.getLogger()
 
@@ -113,22 +132,10 @@ class DjangoCommandSupervisee(SuperviseeSpec):
   def cmdv(self):
     return [ HUE_BIN, self.django_command ]
 
-
-class TimeOutPIDLockFile(PIDLockFile):
-  """A PIDLockFile subclass that passes through a timeout on acquisition."""
-  def __init__(self, lockfile, timeout, **kwargs):
-    PIDLockFile.__init__(self, lockfile, **kwargs)
-    self.timeout = timeout
-
-  def __enter__(self):
-    super(TimeOutPIDLockFile, self).acquire(timeout=self.timeout)
-    return self
-
-
 class Supervisor(threading.Thread):
   """A thread responsible for keeping the supervised subprocess running"""
   # States of the subprocess
-  STATES = (PENDING, RUNNING, FINISHED, ERROR) = range(4)
+  STATES = (PENDING, RUNNING, FINISHED, ERROR) = list(range(4))
 
   def __init__(self, cmdv, **kwargs):
     super(Supervisor, self).__init__()
@@ -176,7 +183,7 @@ class Supervisor(threading.Thread):
           return
 
         LOG.error("Process %s exited abnormally. Restarting it." % (proc_str,))
-    except MyBaseException, ex:
+    except MyBaseException as ex:
       LOG.exception("Uncaught exception. Supervisor exiting.")
       self.state = Supervisor.ERROR
 
@@ -269,19 +276,19 @@ def drop_privileges():
   """
   we_are_root = os.getuid() == 0
   if not we_are_root:
-    print >>sys.stdout, "[INFO] Not running as root, skipping privilege drop"
+    print("[INFO] Not running as root, skipping privilege drop", file=sys.stdout)
     return
 
   try:
     pw = pwd.getpwnam(SETUID_USER)
   except KeyError:
-    print >>sys.stderr, "[ERROR] Couldn't get user information for user " + SETUID_USER
+    print("[ERROR] Couldn't get user information for user " + SETUID_USER, file=sys.stderr)
     raise
 
   try:
     gr = grp.getgrnam(SETGID_GROUP)
   except KeyError:
-    print >>sys.stderr, "[ERROR] Couldn't get group information for group " + SETGID_GROUP
+    print("[ERROR] Couldn't get group information for group " + SETGID_GROUP, file=sys.stderr)
     raise
 
   # gid has to be set first
@@ -305,7 +312,7 @@ def main():
   log_dir = os.path.join(root, options.log_dir)
 
   if options.show_supervisees:
-    for name, supervisee in get_supervisees().iteritems():
+    for name, supervisee in get_supervisees().items():
       if name not in options.supervisee_exclusions:
         print(name)
     sys.exit(0)
@@ -351,7 +358,7 @@ def main():
         }
 
     context.open()
-  os.umask(022)
+  os.umask(0o22)
 
   # Log initialization must come after daemonization, which closes all open files.
   # Log statements before this point goes to stderr.
@@ -359,7 +366,7 @@ def main():
 
   sups = []
   try:
-    for name, supervisee in get_supervisees().iteritems():
+    for name, supervisee in get_supervisees().items():
 
       if name in options.supervisee_exclusions:
         continue
@@ -384,7 +391,7 @@ def main():
       sups.append(sup)
 
     wait_loop(sups, options)
-  except MyBaseException, ex:
+  except MyBaseException as ex:
     LOG.exception("Exception in supervisor main loop")
     shutdown(sups)      # shutdown() exits the process
 

+ 27 - 19
desktop/core/src/desktop/tests.py

@@ -16,10 +16,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import object
 import json
 import logging
 import os
-import StringIO
 import subprocess
 import sys
 import time
@@ -46,7 +50,7 @@ from nose.tools import assert_true, assert_false, assert_equal, assert_not_equal
 from beeswax.conf import HIVE_SERVER_HOST
 from pig.models import PigScript
 from useradmin.models import GroupPermission
-from settings import DATABASES
+from desktop.settings import DATABASES
 
 import desktop
 import desktop.conf
@@ -71,6 +75,10 @@ from desktop.redaction.engine import RedactionPolicy, RedactionRule
 from desktop.views import check_config, home, generate_configspec, load_confs, collect_validation_messages
 from dashboard.conf import HAS_SQL_ENABLED
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from cStringIO import StringIO as string_io
 
 LOG = logging.getLogger(__name__)
 
@@ -79,7 +87,7 @@ def test_home():
   user = User.objects.get(username="test_home")
 
   response = c.get(reverse(home))
-  assert_equal(["notmine", "trash", "mine", "history"], json.loads(response.context[0]['json_tags']).keys())
+  assert_equal(["notmine", "trash", "mine", "history"], list(json.loads(response.context[0]['json_tags']).keys()))
   assert_equal(200, response.status_code)
 
   script, created = PigScript.objects.get_or_create(owner=user)
@@ -268,16 +276,16 @@ def hue_version():
   HUE_VERSION_BAK = HUE_VERSION
 
   try:
-    assert_equal('cdh6.x-SNAPSHOT', _version_from_properties(StringIO.StringIO("""# Autogenerated build properties
+    assert_equal('cdh6.x-SNAPSHOT', _version_from_properties(string_io("""# Autogenerated build properties
 version=3.9.0-cdh5.9.0-SNAPSHOT
 git.hash=f5fbe90b6a1d0c186b0ddc6e65ce5fc8d24725c8
 cloudera.cdh.release=cdh6.x-SNAPSHOT
 cloudera.hash=f5fbe90b6a1d0c186b0ddc6e65ce5fc8d24725c8aaaaa""")))
 
-    assert_false(_version_from_properties(StringIO.StringIO("""# Autogenerated build properties
+    assert_false(_version_from_properties(string_io("""# Autogenerated build properties
 version=3.9.0-cdh5.9.0-SNAPSHOT git.hash=f5fbe90b6a1d0c186b0ddc6e65ce5fc8d24725c8 cloudera.hash=f5fbe90b6a1d0c186b0ddc6e65ce5fc8d24725c8aaaaa""")))
 
-    assert_false(_version_from_properties(StringIO.StringIO('')))
+    assert_false(_version_from_properties(string_io('')))
   finally:
     HUE_VERSION = HUE_VERSION_BAK
 
@@ -303,8 +311,8 @@ def test_prefs():
   # Check multiple values
   c.post('/desktop/api2/user_preferences/elephant', {'set': 'room'})
   response = c.get('/desktop/api2/user_preferences/')
-  assert_true("baz" in json.loads(response.content)['data'].values(), response.content)
-  assert_true("room" in json.loads(response.content)['data'].values(), response.content)
+  assert_true("baz" in list(json.loads(response.content)['data'].values()), response.content)
+  assert_true("room" in list(json.loads(response.content)['data'].values()), response.content)
 
   # Delete everything
   c.post('/desktop/api2/user_preferences/elephant', {'delete': ''})
@@ -349,26 +357,26 @@ def test_paginator():
     assert_equal(page.end_index(), end)
 
   # First page 1-20
-  obj = range(20)
+  obj = list(range(20))
   pgn = Paginator(obj, per_page=20)
   assert_page(pgn.page(1), obj, 1, 20)
 
   # Handle extra data on first page (22 items on a 20-page)
-  obj = range(22)
+  obj = list(range(22))
   pgn = Paginator(obj, per_page=20)
-  assert_page(pgn.page(1), range(20), 1, 20)
+  assert_page(pgn.page(1), list(range(20)), 1, 20)
 
   # Handle total < len(obj). Only works for QuerySet.
   obj = query.QuerySet()
-  obj._result_cache = range(10)
+  obj._result_cache = list(range(10))
   pgn = Paginator(obj, per_page=10)
-  assert_page(pgn.page(1), range(10), 1, 10)
+  assert_page(pgn.page(1), list(range(10)), 1, 10)
 
   # Still works with a normal complete list
-  obj = range(25)
+  obj = list(range(25))
   pgn = Paginator(obj, per_page=20)
-  assert_page(pgn.page(1), range(20), 1, 20)
-  assert_page(pgn.page(2), range(20, 25), 21, 25)
+  assert_page(pgn.page(1), list(range(20)), 1, 20)
+  assert_page(pgn.page(2), list(range(20, 25)), 21, 25)
 
 def test_thread_dump():
   c = make_logged_in_client()
@@ -753,7 +761,7 @@ def test_validate_path():
     reset = desktop.conf.SSL_PRIVATE_KEY.set_for_testing('/tmm/does_not_exist')
     assert_not_equal([], validate_path(desktop.conf.SSL_PRIVATE_KEY, is_dir=True))
     assert_true(False)
-  except Exception, ex:
+  except Exception as ex:
     assert_true('does not exist' in str(ex), ex)
   finally:
     reset()
@@ -865,14 +873,14 @@ def test_cx_Oracle():
   try:
     import cx_Oracle
     return
-  except ImportError, ex:
+  except ImportError as ex:
     if "No module named" in ex.message:
       assert_true(False, "cx_Oracle skipped its build. This happens if "
           "env var ORACLE_HOME or ORACLE_INSTANTCLIENT_HOME is not defined. "
           "So ignore this test failure if your build does not need to work "
           "with an oracle backend.")
 
-class TestStrictRedirection():
+class TestStrictRedirection(object):
 
   def setUp(self):
     self.client = make_logged_in_client()

+ 2 - 1
desktop/core/src/desktop/tests_doc2.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import json
 
 from datetime import datetime
@@ -39,7 +40,7 @@ from useradmin.models import get_default_user_group
 from oozie.models2 import Workflow
 
 
-class MockFs():
+class MockFs(object):
   def __init__(self):
     pass
 

+ 14 - 9
desktop/core/src/desktop/views.py

@@ -15,7 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import StringIO
+from future import standard_library
+standard_library.install_aliases()
 import json
 import logging
 import os
@@ -62,6 +63,10 @@ from desktop.log.access import access_log_level, access_warn, AccessInfo
 from desktop.log import set_all_debug as _set_all_debug, reset_all_debug as _reset_all_debug, get_all_debug as _get_all_debug
 from desktop.models import Settings, hue_version, _get_apps, UserPreferences
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from StringIO import StringIO as string_io
 
 LOG = logging.getLogger(__name__)
 
@@ -77,7 +82,7 @@ def is_alive(request):
 
 def hue(request):
   current_app, other_apps, apps_list = _get_apps(request.user, '')
-  clusters = get_clusters(request.user).values()
+  clusters = list(get_clusters(request.user).values())
 
   return render('hue.mako', request, {
     'apps': apps_list,
@@ -234,7 +239,7 @@ def download_log_view(request):
         response['Content-Disposition'] = 'attachment; filename=hue-logs-%s.zip' % t
         response['Content-Length'] = length
         return response
-      except Exception, e:
+      except Exception as e:
         LOG.exception("Couldn't construct zip file to write logs")
         return log_view(request)
 
@@ -292,7 +297,7 @@ def dump_config(request):
 
   apps = sorted(appmanager.DESKTOP_MODULES, key=lambda app: app.name)
   apps_names = [app.name for app in apps]
-  top_level = sorted(GLOBAL_CONFIG.get().values(), key=lambda obj: apps_names.index(obj.config.key))
+  top_level = sorted(list(GLOBAL_CONFIG.get().values()), key=lambda obj: apps_names.index(obj.config.key))
 
   return render("dump_config.mako", request, dict(
     show_private=show_private,
@@ -304,7 +309,7 @@ def dump_config(request):
 @access_log_level(logging.WARN)
 def threads(request):
   """Dumps out server threads. Useful for debugging."""
-  out = StringIO.StringIO()
+  out = string_io()
   dump_traceback(file=out)
 
   if not is_admin(request.user):
@@ -605,7 +610,7 @@ def _get_config_errors(request, cache=True):
             error['value'] = confvar.get()
 
           error_list.append(error)
-      except Exception, ex:
+      except Exception as ex:
         LOG.exception("Error in config validation by %s: %s" % (module.nice_name, ex))
 
     validate_by_spec(error_list)
@@ -666,7 +671,7 @@ def collect_validation_messages(conf, error_list):
     'remote_data_dir': [('liboozie', )],
     'shell': [()]
   }
-  whitelist_extras = ((sections, name) for sections, name in get_extra_values(conf) if not (name in desktop.conf.APP_BLACKLIST.get() or (name in cm_extras.keys() and sections in cm_extras[name])))
+  whitelist_extras = ((sections, name) for sections, name in get_extra_values(conf) if not (name in desktop.conf.APP_BLACKLIST.get() or (name in list(cm_extras.keys()) and sections in cm_extras[name])))
 
   for sections, name in whitelist_extras:
     the_section = conf
@@ -677,14 +682,14 @@ def collect_validation_messages(conf, error_list):
         the_section = parent[section]
         hierarchy_sections_string += "[" * the_section.depth + section + "]" * the_section.depth + " "
         parent = the_section
-    except KeyError, ex:
+    except KeyError as ex:
       LOG.warn("Section %s not found: %s" % (section, str(ex)))
 
     the_value = ''
     try:
       # the_value may be a section or a value
       the_value = the_section[name]
-    except KeyError, ex:
+    except KeyError as ex:
       LOG.warn("Error in accessing Section or Value %s: %s" % (name, str(ex)))
 
     section_or_value = 'keyvalue'