Эх сурвалжийг харах

HUE-8737 [core] Futurize desktop/libs/liboozie for Python 3.5

Ying Chen 6 жил өмнө
parent
commit
43a84e8184

+ 5 - 3
desktop/libs/liboozie/src/liboozie/conf.py

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import oct
+from builtins import object
 import logging
 import sys
 
@@ -87,7 +89,7 @@ def config_validator(user):
 
   try:
     from oozie.conf import REMOTE_SAMPLE_DIR
-  except Exception, e:
+  except Exception as e:
     LOG.warn('Config check failed because Oozie app not installed: %s' % e)
     return res
 
@@ -123,12 +125,12 @@ def config_validator(user):
     if not sharelib_url:
       res.append((status, _('Oozie Share Lib path is not available')))
 
-    class ConfigMock:
+    class ConfigMock(object):
       def __init__(self, value): self.value = value
       def get(self): return self.value
       def get_fully_qualifying_key(self): return self.value
 
-    for cluster in get_all_hdfs().values():
+    for cluster in list(get_all_hdfs().values()):
       res.extend(validate_path(ConfigMock(sharelib_url), is_dir=True, fs=cluster,
                                message=_('Oozie Share Lib not installed in default location.')))
 

+ 2 - 1
desktop/libs/liboozie/src/liboozie/credentials.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import logging
 
 from django.utils.translation import ugettext as _
@@ -45,7 +46,7 @@ class Credentials(object):
 
   @property
   def class_to_name_credentials(self):
-    return dict((v,k) for k, v in self.credentials.iteritems())
+    return dict((v,k) for k, v in self.credentials.items())
 
   def get_properties(self, hive_properties=None):
     credentials = {}

+ 2 - 1
desktop/libs/liboozie/src/liboozie/credentials_tests.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import logging
 
 from nose.tools import assert_equal, assert_true
@@ -27,7 +28,7 @@ from liboozie.credentials import Credentials
 LOG = logging.getLogger(__name__)
 
 
-class TestCredentials():
+class TestCredentials(object):
   CREDENTIALS = {
     "hcat": "org.apache.oozie.action.hadoop.HCatCredentials",
     "hive2": "org.apache.oozie.action.hadoop.Hive2Credentials",

+ 2 - 1
desktop/libs/liboozie/src/liboozie/oozie_api.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import logging
 import posixpath
 
@@ -345,6 +346,6 @@ class OozieApi(object):
       nominal_end=2013-06-23T00:01Z
     """
     params = self._get_params()
-    params['filter'] = ';'.join(['%s=%s' % (key, val) for key, val in kwargs.iteritems()])
+    params['filter'] = ';'.join(['%s=%s' % (key, val) for key, val in kwargs.items()])
     resp = self._root.get('sla', params)
     return resp['slaSummaryList']

+ 5 - 3
desktop/libs/liboozie/src/liboozie/oozie_api_tests.py

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import oct
+from builtins import object
 import atexit
 import getpass
 import logging
@@ -229,7 +231,7 @@ class OozieServerProvider(object):
               break
             time.sleep(sleep)
             sleep *= 2
-          except Exception, e:
+          except Exception as e:
             LOG.info('Oozie server status not NORMAL yet: %s - %s' % (status, e))
             time.sleep(sleep)
             sleep *= 2
@@ -277,7 +279,7 @@ class TestOozieWorkspace(object):
       assert_false('The permissions of workspace' in resp.content, resp)
 
       self.cluster.fs.mkdir(REMOTE_SAMPLE_DIR.get())
-      assert_equal(oct(040755), oct(self.cluster.fs.stats(REMOTE_SAMPLE_DIR.get())["mode"]))
+      assert_equal(oct(0o40755), oct(self.cluster.fs.stats(REMOTE_SAMPLE_DIR.get())["mode"]))
       resp = self.cli.get('/desktop/debug/check_config')
       assert_true('The permissions of workspace' in resp.content, resp)
 
@@ -292,7 +294,7 @@ class TestOozieWorkspace(object):
 
       # Add write permission to Others
       response = self.cli.post("/filebrowser/chmod", kwargs)
-      assert_equal(oct(040757), oct(self.cluster.fs.stats(REMOTE_SAMPLE_DIR.get())["mode"]))
+      assert_equal(oct(0o40757), oct(self.cluster.fs.stats(REMOTE_SAMPLE_DIR.get())["mode"]))
 
       resp = self.cli.get('/desktop/debug/check_config')
       assert_false('The permissions of workspace' in resp.content, resp)

+ 11 - 10
desktop/libs/liboozie/src/liboozie/submission2.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import errno
 import logging
 import os
@@ -190,7 +191,7 @@ class Submission(object):
     try:
       if not deployment_dir:
         deployment_dir = self._create_deployment_dir()
-    except Exception, ex:
+    except Exception as ex:
       msg = _("Failed to create deployment directory: %s" % ex)
       LOG.exception(msg)
       raise PopupException(message=msg, detail=str(ex))
@@ -373,7 +374,7 @@ STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.jo
             hive_site_lib = Hdfs.join(deployment_dir + '/lib/', 'hive-site.xml')
             hive_site_content = get_hive_site_content()
             if not self.fs.do_as_user(self.user, self.fs.exists, hive_site_lib) and hive_site_content:
-              self.fs.do_as_user(self.user, self.fs.create, hive_site_lib, overwrite=True, permission=0700, data=smart_str(hive_site_content))
+              self.fs.do_as_user(self.user, self.fs.create, hive_site_lib, overwrite=True, permission=0o700, data=smart_str(hive_site_content))
           if action.data['type'] in ('sqoop', 'sqoop-document'):
             if CONFIG_JDBC_LIBS_PATH.get() and CONFIG_JDBC_LIBS_PATH.get() not in self.properties.get('oozie.libpath', ''):
               LOG.debug("Adding to oozie.libpath %s" % CONFIG_JDBC_LIBS_PATH.get())
@@ -472,7 +473,7 @@ STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.jo
               ('hive2.jdbc.url', hive_jdbc_url),
               ('hive2.server.principal', updated_hive_principal)
             ]
-          except Exception, ex:
+          except Exception as ex:
             msg = 'Failed to update the Hive JDBC URL from %s action properties: %s' % (action.data['type'], str(ex))
             LOG.error(msg)
             raise PopupException(message=_(msg), detail=str(ex))
@@ -516,7 +517,7 @@ STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.jo
         msg = _("Path is not a directory: %s.") % (path,)
         LOG.error(msg)
         raise Exception(msg)
-    except IOError, ex:
+    except IOError as ex:
       if ex.errno != errno.ENOENT:
         msg = _("Error accessing directory '%s': %s.") % (path, ex)
         LOG.exception(msg)
@@ -537,7 +538,7 @@ STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.jo
     """
 
     self._create_file(deployment_dir, self.job.XML_FILE_NAME, oozie_xml)
-    self._create_file(deployment_dir, 'job.properties', data='\n'.join(['%s=%s' % (key, val) for key, val in oozie_properties.iteritems()]))
+    self._create_file(deployment_dir, 'job.properties', data='\n'.join(['%s=%s' % (key, val) for key, val in oozie_properties.items()]))
 
     # List jar files
     files = []
@@ -587,7 +588,7 @@ STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.jo
       path = self.job.deployment_dir
       if self._do_as(self.user.username , self.fs.exists, path):
         self._do_as(self.user.username , self.fs.rmtree, path)
-    except Exception, ex:
+    except Exception as ex:
       LOG.warn("Failed to clean up workflow deployment directory for %s (owner %s). Caused by: %s", self.job.name, self.user, ex)
 
   def _is_workflow(self):
@@ -601,9 +602,9 @@ STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.jo
   def _create_file(self, deployment_dir, file_name, data, do_as=False):
     file_path = self.fs.join(deployment_dir, file_name)
     if do_as:
-      self.fs.do_as_user(self.user, self.fs.create, file_path, overwrite=True, permission=0644, data=smart_str(data))
+      self.fs.do_as_user(self.user, self.fs.create, file_path, overwrite=True, permission=0o644, data=smart_str(data))
     else:
-      self.fs.create(file_path, overwrite=True, permission=0644, data=smart_str(data))
+      self.fs.create(file_path, overwrite=True, permission=0o644, data=smart_str(data))
     LOG.debug("Created/Updated %s" % (file_path,))
 
   def _generate_altus_action_script(self, service, command, arguments, auth_key_id, auth_key_secret):
@@ -819,5 +820,5 @@ def create_directories(fs, directory_list=[]):
         # Home is 755
         fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir)
       # Shared by all the users
-      fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777)
-      fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
+      fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 0o1777)
+      fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 0o1777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491

+ 9 - 8
desktop/libs/liboozie/src/liboozie/submittion.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import errno
 import logging
 import os
@@ -147,7 +148,7 @@ class Submission(object):
   def deploy(self):
     try:
       deployment_dir = self._create_deployment_dir()
-    except Exception, ex:
+    except Exception as ex:
       msg = _("Failed to create deployment directory: %s" % ex)
       LOG.exception(msg)
       raise PopupException(message=msg, detail=str(ex))
@@ -234,7 +235,7 @@ class Submission(object):
       path = Hdfs.join(REMOTE_DEPLOYMENT_DIR.get(), '_%s_-oozie-%s-%s' % (self.user.username, self.job.id, time.time()))
       # Shared coords or bundles might not have any existing workspaces
       if self.fs.exists(self.job.deployment_dir):
-        self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user, dir_mode=0711)
+        self.fs.copy_remote_dir(self.job.deployment_dir, path, owner=self.user, dir_mode=0o711)
       else:
         self._create_dir(path)
     else:
@@ -242,7 +243,7 @@ class Submission(object):
       self._create_dir(path)
     return path
 
-  def _create_dir(self, path, perms=0711):
+  def _create_dir(self, path, perms=0o711):
     """
     Return the directory in HDFS, creating it if necessary.
     """
@@ -252,7 +253,7 @@ class Submission(object):
         msg = _("Path is not a directory: %s.") % (path,)
         LOG.error(msg)
         raise Exception(msg)
-    except IOError, ex:
+    except IOError as ex:
       if ex.errno != errno.ENOENT:
         msg = _("Error accessing directory '%s': %s.") % (path, ex)
         LOG.exception(msg)
@@ -271,7 +272,7 @@ class Submission(object):
     This should run as the workflow user.
     """
     xml_path = self.fs.join(deployment_dir, self.job.get_application_filename())
-    self.fs.create(xml_path, overwrite=True, permission=0644, data=smart_str(oozie_xml))
+    self.fs.create(xml_path, overwrite=True, permission=0o644, data=smart_str(oozie_xml))
     LOG.debug("Created %s" % (xml_path,))
 
     # List jar files
@@ -309,7 +310,7 @@ class Submission(object):
       path = self.job.deployment_dir
       if self._do_as(self.user.username , self.fs.exists, path):
         self._do_as(self.user.username , self.fs.rmtree, path)
-    except Exception, ex:
+    except Exception as ex:
       LOG.warn("Failed to clean up workflow deployment directory for "
                "%s (owner %s). Caused by: %s",
                self.job.name, self.user, ex)
@@ -334,5 +335,5 @@ def create_directories(fs, directory_list=[]):
         # Home is 755
         fs.do_as_user(fs.DEFAULT_USER, fs.create_home_dir, remote_home_dir)
       # Shared by all the users
-      fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 01777)
-      fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 01777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491
+      fs.do_as_user(fs.DEFAULT_USER, fs.mkdir, directory, 0o1777)
+      fs.do_as_user(fs.DEFAULT_USER, fs.chmod, directory, 0o1777) # To remove after https://issues.apache.org/jira/browse/HDFS-3491

+ 9 - 7
desktop/libs/liboozie/src/liboozie/submittion2_tests.py

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+from builtins import object
 import logging
 
 from django.contrib.auth.models import User
@@ -75,7 +77,7 @@ def test_copy_files():
     cluster.fs.create(deployment_dir + '/' + jar_5)
     cluster.fs.create(deployment_dir + '/' + jar_6)
 
-    class MockJob():
+    class MockJob(object):
       XML_FILE_NAME = 'workflow.xml'
 
       def __init__(self):
@@ -116,7 +118,7 @@ def test_copy_files():
       assert_true(jar_2 in submission.properties['oozie.libpath'])
       assert_true(jar_3 in submission.properties['oozie.libpath'])
       assert_true(jar_4 in submission.properties['oozie.libpath'])
-      print deployment_dir + '/' + jar_5
+      print(deployment_dir + '/' + jar_5)
       assert_true((deployment_dir + '/' + jar_5) in submission.properties['oozie.libpath'], submission.properties['oozie.libpath'])
       assert_true((deployment_dir + '/' + jar_6) in submission.properties['oozie.libpath'], submission.properties['oozie.libpath'])
     else:
@@ -165,14 +167,14 @@ def test_copy_files():
       LOG.exception('failed to remove %s' % prefix)
 
 
-class MockFs():
+class MockFs(object):
   def __init__(self, logical_name=None):
 
     self.fs_defaultfs = 'hdfs://curacao:8020'
     self.logical_name = logical_name if logical_name else ''
 
 
-class MockJt():
+class MockJt(object):
   def __init__(self, logical_name=None):
 
     self.logical_name = logical_name if logical_name else ''
@@ -320,7 +322,7 @@ oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pig
 
   def test_update_credentials_from_hive_action(self):
 
-    class TestJob():
+    class TestJob(object):
       XML_FILE_NAME = 'workflow.xml'
 
       def __init__(self):
@@ -371,7 +373,7 @@ oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pig
 
   def test_update_credentials_from_hive_action_when_jdbc_url_is_variable(self):
 
-    class TestJob():
+    class TestJob(object):
       XML_FILE_NAME = 'workflow.xml'
 
       def __init__(self):
@@ -411,7 +413,7 @@ oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pig
 
   def test_generate_altus_action_start_cluster(self):
 
-    class TestJob():
+    class TestJob(object):
       XML_FILE_NAME = 'workflow.xml'
 
       def __init__(self):

+ 5 - 4
desktop/libs/liboozie/src/liboozie/submittion_tests.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import logging
 
 from django.contrib.auth.models import User
@@ -64,11 +65,11 @@ def test_copy_files():
     cluster.fs.create(jar_3)
     cluster.fs.create(jar_4)
 
-    class MockNode():
+    class MockNode(object):
       def __init__(self, jar_path):
         self.jar_path = jar_path
 
-    class MockJob():
+    class MockJob(object):
       def __init__(self):
         self.node_list = [
             MockNode(jar_1),
@@ -127,14 +128,14 @@ def test_copy_files():
       LOG.exception('failed to remove %s' % prefix)
 
 
-class MockFs():
+class MockFs(object):
   def __init__(self, logical_name=None):
 
     self.fs_defaultfs = 'hdfs://curacao:8020'
     self.logical_name = logical_name if logical_name else ''
 
 
-class MockJt():
+class MockJt(object):
   def __init__(self, logical_name=None):
 
     self.logical_name = logical_name if logical_name else ''

+ 19 - 9
desktop/libs/liboozie/src/liboozie/types.py

@@ -21,12 +21,17 @@ Oozie API classes.
 This is mostly just codifying the datastructure of the Oozie REST API.
 http://incubator.apache.org/oozie/docs/3.2.0-incubating/docs/WebServicesAPI.html
 """
+from __future__ import division
 
+from future import standard_library
+standard_library.install_aliases()
+from past.utils import old_div
+from builtins import object
 import logging
 import re
+import sys
 import time
 
-from cStringIO import StringIO
 from time import mktime
 
 from desktop.lib import i18n
@@ -41,6 +46,11 @@ from django.urls import reverse
 
 from desktop.auth.backend import is_admin
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+else:
+  from cStringIO import StringIO as string_io
+
 LOG = logging.getLogger(__name__)
 
 
@@ -158,10 +168,10 @@ class WorkflowAction(Action):
       self.retries = int(self.retries)
 
     if self.conf:
-      xml = StringIO(i18n.smart_str(self.conf))
+      xml = string_io(i18n.smart_str(self.conf))
       try:
         self.conf_dict = hadoop.confparse.ConfParse(xml)
-      except Exception, e:
+      except Exception as e:
         LOG.error('Failed to parse XML configuration for Workflow action %s: %s' % (self.name, e))
         self.conf_dict = {}
     else:
@@ -234,7 +244,7 @@ class CoordinatorAction(Action):
       self.lastModifiedTime = parse_timestamp(self.lastModifiedTime)
 
     if self.runConf:
-      xml = StringIO(i18n.smart_str(self.runConf))
+      xml = string_io(i18n.smart_str(self.runConf))
       self.conf_dict = hadoop.confparse.ConfParse(xml)
     else:
       self.conf_dict = {}
@@ -282,7 +292,7 @@ class BundleAction(Action):
     self.name = self.coordJobName
 
     if self.conf:
-      xml = StringIO(i18n.smart_str(self.conf))
+      xml = string_io(i18n.smart_str(self.conf))
       self.conf_dict = hadoop.confparse.ConfParse(xml)
     else:
       self.conf_dict = {}
@@ -297,7 +307,7 @@ class BundleAction(Action):
     end = mktime(parse_timestamp(self.endTime))
 
     if end != start:
-      progress = min(int((1 - (end - next) / (end - start)) * 100), 100)
+      progress = min(int((1 - old_div((end - next), (end - start))) * 100), 100)
     else:
       progress = 100
 
@@ -336,7 +346,7 @@ class Job(object):
 
     self.actions = [Action.create(self.ACTION, act_dict) for act_dict in self.actions]
     if self.conf is not None:
-      xml = StringIO(i18n.smart_str(self.conf))
+      xml = string_io(i18n.smart_str(self.conf))
       self.conf_dict = hadoop.confparse.ConfParse(xml)
     else:
       self.conf_dict = {}
@@ -568,14 +578,14 @@ class Coordinator(Job):
     end = mktime(self.endTime)
 
     if end != start:
-      progress = min(int((1 - (end - next) / (end - start)) * 100), 100)
+      progress = min(int((1 - old_div((end - next), (end - start))) * 100), 100)
     else:
       progress = 100
 
     # Manage case of a rerun
     action_count = float(len(self.actions))
     if action_count != 0 and progress == 100:
-      progress = int(sum([action.is_finished() for action in self.actions]) / action_count * 100)
+      progress = int(old_div(sum([action.is_finished() for action in self.actions]), action_count * 100))
 
     return progress
 

+ 22 - 12
desktop/libs/liboozie/src/liboozie/utils.py

@@ -18,14 +18,15 @@
 """
 Misc helper functions
 """
+from __future__ import print_function
 
-try:
-  from cStringIO import StringIO
-except ImportError:
-  from StringIO import StringIO
+from future import standard_library
+standard_library.install_aliases()
+from past.builtins import basestring
 
 import logging
 import re
+import sys
 import time
 
 from datetime import datetime
@@ -33,6 +34,15 @@ from dateutil.parser import parse
 from time import strftime
 from xml.sax.saxutils import escape
 
+if sys.version_info[0] > 2:
+  from io import StringIO as string_io
+  new_str = str
+else:
+  try:
+    from cStringIO import StringIO as string_io
+  except:
+    from StringIO import StringIO as string_io
+  new_str = unicode
 
 LOG = logging.getLogger(__name__)
 _NAME_REGEX = re.compile('^[a-zA-Z][\-_a-zA-Z0-0]*$')
@@ -69,14 +79,14 @@ def config_gen(dic):
   """
   config_gen(dic) -> xml for Oozie workflow configuration
   """
-  sio = StringIO()
-  print >> sio, '<?xml version="1.0" encoding="UTF-8"?>'
-  print >> sio, "<configuration>"
+  sio = string_io()
+  print('<?xml version="1.0" encoding="UTF-8"?>', file=sio)
+  print("<configuration>", file=sio)
   # if dic's key contains <,>,& then it will be escaped and if dic's value contains ']]>' then ']]>' will be stripped
-  for k, v in dic.iteritems():
-    print >> sio, "<property>\n  <name>%s</name>\n  <value><![CDATA[%s]]></value>\n</property>\n" \
-        % (escape(k), v.replace(']]>', '') if isinstance(v, basestring) else v)
-  print >>sio, "</configuration>"
+  for k, v in dic.items():
+    print("<property>\n  <name>%s</name>\n  <value><![CDATA[%s]]></value>\n</property>\n" \
+        % (escape(k), v.replace(']]>', '') if isinstance(v, basestring) else v), file=sio)
+  print("</configuration>", file=sio)
   sio.flush()
   sio.seek(0)
   return sio.read()
@@ -90,7 +100,7 @@ def format_time(time, format='%d %b %Y %H:%M:%S'):
     return ''
 
   fmt_time = None
-  if type(time) == unicode:
+  if type(time) == new_str:
     return time
   else:
     try: