Forráskód Böngészése

HUE-8737 [core] Futurize desktop/libs/metadata for Python 3.5

Ying Chen 6 éve
szülő
commit
abb5760e03

+ 1 - 1
desktop/libs/metadata/src/metadata/analytic_db_api.py

@@ -32,7 +32,7 @@ def error_handler(view_fn):
   def decorator(*args, **kwargs):
     try:
       return view_fn(*args, **kwargs)
-    except Exception, e:
+    except Exception as e:
       LOG.exception(e)
       response = {
         'status': -1,

+ 1 - 1
desktop/libs/metadata/src/metadata/catalog/atlas_client.py

@@ -127,7 +127,7 @@ class AtlasApi(Api):
       nav_entity['classifications'] = atlas_entity['classifications']
       for atlas_classification in atlas_entity['classifications']:
         if 'attributes' in atlas_classification:
-          for key, value in atlas_classification['attributes'].iteritems():
+          for key, value in atlas_classification['attributes'].items():
             nav_entity['properties'][key] = value
 
     return nav_entity

+ 1 - 0
desktop/libs/metadata/src/metadata/catalog/base.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 from django.utils.translation import ugettext as _
 
 from desktop.lib.exceptions_renderable import PopupException

+ 16 - 16
desktop/libs/metadata/src/metadata/catalog/navigator_client.py

@@ -197,7 +197,7 @@ class NavigatorApi(Api):
           }
         }
 
-      auto_field_facets = ["tags", "type"] + f.keys()
+      auto_field_facets = ["tags", "type"] + list(f.keys())
       query_s = (query_s.strip() if query_s else '') + '*'
 
       query_s = query_s.replace('tag:', 'tags:').replace('classification:', 'tags:')
@@ -275,7 +275,7 @@ class NavigatorApi(Api):
       response['results'] = list(islice(self._secure_results(response['results']), limit)) # Apply Sentry perms
 
       return response
-    except RestException, e:
+    except RestException as e:
       LOG.error('Failed to search for entities with search query: %s' % json.dumps(body))
       if e.code == 401:
         raise CatalogAuthException(_('Failed to authenticate.'))
@@ -355,7 +355,7 @@ class NavigatorApi(Api):
       response = list(islice(self._secure_results(response), limit)) # Apply Sentry perms
 
       return response
-    except RestException, e:
+    except RestException as e:
       LOG.error('Failed to search for entities with search query: %s' % query_s)
       if e.code == 401:
         raise CatalogAuthException(_('Failed to authenticate.'))
@@ -393,7 +393,7 @@ class NavigatorApi(Api):
   def suggest(self, prefix=None):
     try:
       return self._root.get('interactive/suggestions?query=%s' % (prefix or '*'))
-    except RestException, e:
+    except RestException as e:
       msg = 'Failed to search for entities with search query: %s' % prefix
       LOG.error(msg)
       raise CatalogApiException(e.message)
@@ -413,10 +413,10 @@ class NavigatorApi(Api):
         'deleted': 'false'
       }
 
-      for key, value in filters.items():
+      for key, value in list(filters.items()):
         query_filters[key] = value
 
-      filter_query = 'AND'.join('(%s:%s)' % (key, value) for key, value in query_filters.items())
+      filter_query = 'AND'.join('(%s:%s)' % (key, value) for key, value in list(query_filters.items()))
       filter_query = '%(type)s AND %(filter_query)s' % {
         'type': '(type:%s)' % 'TABLE OR type:VIEW' if type == 'TABLE' else type, # Impala does not always say that a table is actually a view
         'filter_query': filter_query
@@ -440,7 +440,7 @@ class NavigatorApi(Api):
         raise CatalogApiException('Found more than 1 entity with query filters: %s' % str(query_filters))
 
       return response[0]
-    except RestException, e:
+    except RestException as e:
       msg = 'Failed to find entity: %s' % str(e)
       LOG.error(msg)
       raise CatalogApiException(e.message)
@@ -453,7 +453,7 @@ class NavigatorApi(Api):
     """
     try:
       return self._root.get('entities/%s' % entity_id, headers=self.__headers, params=self.__params)
-    except RestException, e:
+    except RestException as e:
       msg = 'Failed to get entity %s: %s' % (entity_id, str(e))
       LOG.error(msg)
       raise CatalogApiException(e.message)
@@ -476,7 +476,7 @@ class NavigatorApi(Api):
       data = json.dumps(properties)
 
       return self._root.put('entities/%(identity)s' % entity, params=self.__params, data=data, contenttype=_JSON_CONTENT_TYPE, allow_redirects=True, clear_cookies=True)
-    except RestException, e:
+    except RestException as e:
       msg = 'Failed to update entity %s: %s' % (entity['identity'], e)
       LOG.error(msg)
       raise CatalogApiException(e.message)
@@ -544,7 +544,7 @@ class NavigatorApi(Api):
       )
 
       return self._root.get('lineage', headers=self.__headers, params=params)
-    except RestException, e:
+    except RestException as e:
       msg = 'Failed to get lineage for entity ID %s: %s' % (entity_id, str(e))
       LOG.error(msg)
       raise CatalogApiException(e.message)
@@ -554,7 +554,7 @@ class NavigatorApi(Api):
     try:
       data = json.dumps({'name': namespace, 'description': description})
       return self._root.post('models/namespaces/', data=data, contenttype=_JSON_CONTENT_TYPE, clear_cookies=True)
-    except RestException, e:
+    except RestException as e:
       msg = 'Failed to create namespace: %s' % namespace
       LOG.error(msg)
       raise CatalogApiException(e.message)
@@ -563,7 +563,7 @@ class NavigatorApi(Api):
   def get_namespace(self, namespace):
     try:
       return self._root.get('models/namespaces/%(namespace)s' % {'namespace': namespace})
-    except RestException, e:
+    except RestException as e:
       msg = 'Failed to get namespace: %s' % namespace
       LOG.error(msg)
       raise CatalogApiException(e.message)
@@ -573,7 +573,7 @@ class NavigatorApi(Api):
     try:
       data = json.dumps(properties)
       return self._root.post('models/namespaces/%(namespace)s/properties' % {'namespace': namespace}, data=data, contenttype=_JSON_CONTENT_TYPE, clear_cookies=True)
-    except RestException, e:
+    except RestException as e:
       msg = 'Failed to create namespace %s property' % namespace
       LOG.error(msg)
       raise CatalogApiException(e.message)
@@ -582,7 +582,7 @@ class NavigatorApi(Api):
   def get_namespace_properties(self, namespace):
     try:
       return self._root.get('models/namespaces/%(namespace)s/properties' % {'namespace': namespace})
-    except RestException, e:
+    except RestException as e:
       msg = 'Failed to create namespace %s property' % namespace
       LOG.error(msg)
       raise CatalogApiException(e.message)
@@ -592,7 +592,7 @@ class NavigatorApi(Api):
     try:
       data = json.dumps(properties)
       return self._root.post('models/packages/nav/classes/%(class)s/properties' % {'class': clazz}, data=data, contenttype=_JSON_CONTENT_TYPE, clear_cookies=True)
-    except RestException, e:
+    except RestException as e:
       msg = 'Failed to map class %s property' % clazz
       LOG.error(msg)
       raise CatalogApiException(e.message)
@@ -601,7 +601,7 @@ class NavigatorApi(Api):
   def get_model_properties_mapping(self):
     try:
       return self._root.get('models/properties/mappings')
-    except RestException, e:
+    except RestException as e:
       msg = 'Failed to get models properties mappings'
       LOG.error(msg)
       raise CatalogApiException(e.message)

+ 3 - 2
desktop/libs/metadata/src/metadata/catalog/navigator_client_tests.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import logging
 
 from nose.plugins.skip import SkipTest
@@ -37,7 +38,7 @@ from metadata.catalog.navigator_client import NavigatorApi
 LOG = logging.getLogger(__name__)
 
 
-class MockedRoot():
+class MockedRoot(object):
   def get(self, relpath=None, params=None, headers=None, clear_cookies=False):
     if relpath == 'entities' and params and params[0] and params[0][0] == 'query' and params[0][1] and params[0][1].startswith('clusterName:'):
       return [{'sourceId': 1}, {'identity': 2}]
@@ -45,7 +46,7 @@ class MockedRoot():
       return params
 
 
-class NavigatorClientTest:
+class NavigatorClientTest(object):
   integration = True
 
   @classmethod

+ 9 - 8
desktop/libs/metadata/src/metadata/catalog_api.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import next
 import json
 import logging
 import re
@@ -54,20 +55,20 @@ def error_handler(view_fn):
         return view_fn(*args, **kwargs)
       else:
         raise MetadataApiException('Catalog API is not configured.')
-    except Http404, e:
+    except Http404 as e:
       raise e
-    except CatalogEntityDoesNotExistException, e:
+    except CatalogEntityDoesNotExistException as e:
       response['message'] = e.message
       status = 404
-    except CatalogAuthException, e:
+    except CatalogAuthException as e:
       response['message'] = force_unicode(e.message)
       status = 403
-    except CatalogApiException, e:
+    except CatalogApiException as e:
       try:
         response['message'] = json.loads(e.message)
       except Exception:
         response['message'] = force_unicode(e.message)
-    except Exception, e:
+    except Exception as e:
       message = force_unicode(e)
       response['message'] = message
       LOG.exception(message)
@@ -106,12 +107,12 @@ def search_entities_interactive(request):
   )
 
   if response.get('facets'): # Remove empty facets
-    for fname, fvalues in response['facets'].items():
+    for fname, fvalues in list(response['facets'].items()):
       # Should be a CATALOG option at some point for hidding table with no access / asking for access.
       if interface == 'navigator' and NAVIGATOR.APPLY_SENTRY_PERMISSIONS.get():
         fvalues = []
       else:
-        fvalues = sorted([(k, v) for k, v in fvalues.items() if v > 0], key=lambda n: n[1], reverse=True)
+        fvalues = sorted([(k, v) for k, v in list(fvalues.items()) if v > 0], key=lambda n: n[1], reverse=True)
       response['facets'][fname] = OrderedDict(fvalues)
       if ':' in query_s and not response['facets'][fname]:
         del response['facets'][fname]
@@ -190,7 +191,7 @@ def _augment_highlighting(query_s, records):
       name = _highlight(term, name)
       if record.get('tags'):
         _highlight_tags(record, term)
-    for fname, fval in fs.iteritems(): # e.g. owner:<em>hu</em>e
+    for fname, fval in fs.items(): # e.g. owner:<em>hu</em>e
       if record.get(fname, ''):
         if fname == 'tags':
           _highlight_tags(record, fval)

+ 2 - 1
desktop/libs/metadata/src/metadata/catalog_tests.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import logging
 import json
 
@@ -276,7 +277,7 @@ class TestNavigator(object):
 
   def _format_json_body(self, post_dict):
     json_dict = {}
-    for key, value in post_dict.items():
+    for key, value in list(post_dict.items()):
       json_dict[key] = json.dumps(value)
     return json_dict
 

+ 1 - 1
desktop/libs/metadata/src/metadata/dataeng_api.py

@@ -34,7 +34,7 @@ def error_handler(view_fn):
   def decorator(*args, **kwargs):
     try:
       return view_fn(*args, **kwargs)
-    except Exception, e:
+    except Exception as e:
       LOG.exception(e)
       response = {
         'status': -1,

+ 2 - 2
desktop/libs/metadata/src/metadata/manager_api.py

@@ -51,12 +51,12 @@ def error_handler(view_fn):
         return view_fn(*args, **kwargs)
       else:
         raise CatalogApiException('Navigator API is not configured.')
-    except CatalogApiException, e:
+    except CatalogApiException as e:
       try:
         response['message'] = json.loads(e.message)
       except Exception:
         response['message'] = force_unicode(e.message)
-    except Exception, e:
+    except Exception as e:
       message = force_unicode(e)
       response['message'] = message
       LOG.exception(message)

+ 21 - 14
desktop/libs/metadata/src/metadata/manager_client.py

@@ -16,11 +16,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
 import base64
 import json
 import logging
-import urllib
-import urllib2
+import sys
 
 from django.core.cache import cache
 from django.utils.translation import ugettext as _
@@ -32,6 +34,11 @@ from desktop.lib.i18n import smart_unicode
 from metadata.conf import MANAGER, get_navigator_auth_username, get_navigator_auth_password
 
 
+if sys.version_info[0] > 2:
+  from urllib.parse import quote as urllib_quote
+else:
+  from urllib import quote as urllib_quote
+
 LOG = logging.getLogger(__name__)
 VERSION = 'v19'
 
@@ -78,7 +85,7 @@ class ManagerApi(object):
       })['items']
 
       return service_name in services
-    except RestException, e:
+    except RestException as e:
       raise ManagerApiException(e)
 
 
@@ -116,7 +123,7 @@ class ManagerApi(object):
             'shs_server_name': shs_server_name
           }, params={'view': 'full'})['items']
           return shs_server_hostId, shs_server_configs
-    except Exception, e:
+    except Exception as e:
       LOG.warn("Check Spark History Server via ManagerApi: %s" % e)
 
     return None, None
@@ -176,7 +183,7 @@ class ManagerApi(object):
 
       LOG.info(params)
       return self._root.get('tools/echo', params=params)
-    except RestException, e:
+    except RestException as e:
       raise ManagerApiException(e)
 
 
@@ -188,7 +195,7 @@ class ManagerApi(object):
       brokers_hosts = [host['hostname'] + ':9092' for host in hosts]
 
       return ','.join(brokers_hosts)
-    except RestException, e:
+    except RestException as e:
       raise ManagerApiException(e)
 
 
@@ -203,7 +210,7 @@ class ManagerApi(object):
       master_host = self._root.get('hosts/%(hostId)s' % master['hostRef'])
 
       return master_host['hostname']
-    except RestException, e:
+    except RestException as e:
       raise ManagerApiException(e)
 
 
@@ -213,7 +220,7 @@ class ManagerApi(object):
       root = Resource(client)
 
       return root.get('/api/topics')
-    except RestException, e:
+    except RestException as e:
       raise ManagerApiException(e)
 
 
@@ -223,7 +230,7 @@ class ManagerApi(object):
     roleConfigGroup = [role['roleConfigGroupRef']['roleConfigGroupName'] for role in self._get_roles(cluster['name'], service, 'AGENT')]
     data = {
       u'items': [{
-        u'url': u'/api/v8/clusters/%(cluster_name)s/services/%(service)s/roleConfigGroups/%(roleConfigGroups)s/config?message=Updated%20service%20and%20role%20type%20configurations.'.replace('%(cluster_name)s', urllib.quote(cluster['name'])).replace('%(service)s', service).replace('%(roleConfigGroups)s', roleConfigGroup[0]),
+        u'url': u'/api/v8/clusters/%(cluster_name)s/services/%(service)s/roleConfigGroups/%(roleConfigGroups)s/config?message=Updated%20service%20and%20role%20type%20configurations.'.replace('%(cluster_name)s', urllib_quote(cluster['name'])).replace('%(service)s', service).replace('%(roleConfigGroups)s', roleConfigGroup[0]),
         u'body': {
           u'items': [
             {u'name': config_name, u'value': config_value}
@@ -254,7 +261,7 @@ class ManagerApi(object):
 
       hosts = self._root.get('hosts')['items']
       return [host for host in hosts if host['hostId'] in hosts_ids]
-    except RestException, e:
+    except RestException as e:
       raise ManagerApiException(e)
 
 
@@ -281,7 +288,7 @@ class ManagerApi(object):
             data=json.dumps({"items": roles}),
             contenttype="application/json"
         )
-    except RestException, e:
+    except RestException as e:
       raise ManagerApiException(e)
 
 
@@ -297,14 +304,14 @@ class ManagerApi(object):
             data=json.dumps({"items": roles}),
             contenttype="application/json"
         )
-    except RestException, e:
+    except RestException as e:
       raise ManagerApiException(e)
 
 
   def batch(self, items):
     try:
       return self._root.post('batch', data=json.dumps(items), contenttype='application/json')
-    except RestException, e:
+    except RestException as e:
       raise ManagerApiException(e)
 
 
@@ -367,7 +374,7 @@ class ManagerApi(object):
               if config['relatedName'] == key:
                 return config['value']
 
-    except Exception, e:
+    except Exception as e:
       LOG.warn("Get Impala Daemon API configurations via ManangerAPI: %s" % e)
 
     return None

+ 1 - 1
desktop/libs/metadata/src/metadata/metadata_sites.py

@@ -80,7 +80,7 @@ def _parse_sites():
 def _parse_property(file_path):
   try:
     return dict(line.strip().rsplit('=', 1) for line in open(file_path) if '=' in line)
-  except IOError, err:
+  except IOError as err:
     if err.errno != errno.ENOENT:
       LOG.error('Cannot read from "%s": %s' % (file_path, err))
     return {}

+ 4 - 2
desktop/libs/metadata/src/metadata/metadata_sites_tests.py

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import
+from builtins import object
 import logging
 import os
 import shutil
@@ -22,14 +24,14 @@ import tempfile
 
 from nose.tools import assert_equal
 
-import metadata_sites
+from . import metadata_sites
 from metadata.conf import NAVIGATOR
 from metadata.metadata_sites import get_navigator_server_url
 
 LOG = logging.getLogger(__name__)
 
 
-class TestReadConfiguration:
+class TestReadConfiguration(object):
 
   def test_navigator_site(self):
     tmpdir = tempfile.mkdtemp()

+ 10 - 9
desktop/libs/metadata/src/metadata/optimizer_api.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import base64
 import json
 import logging
@@ -44,7 +45,7 @@ try:
   from beeswax.design import hql_query
 
   from metastore.views import _get_db
-except ImportError, e:
+except ImportError as e:
   LOG.warn("Hive lib not enabled")
 
 
@@ -52,21 +53,21 @@ def error_handler(view_fn):
   def decorator(*args, **kwargs):
     try:
       return view_fn(*args, **kwargs)
-    except Http404, e:
+    except Http404 as e:
       raise e
-    except NavOptException, e:
+    except NavOptException as e:
       LOG.exception(e)
       response = {
         'status': -1,
         'message': e.message
       }
-    except MissingSentryPrivilegeException, e:
+    except MissingSentryPrivilegeException as e:
       LOG.exception(e)
       response = {
         'status': -1,
         'message': 'Missing privileges for %s' % force_unicode(str(e))
       }
-    except Exception, e:
+    except Exception as e:
       LOG.exception(e)
       response = {
         'status': -1,
@@ -315,7 +316,7 @@ def _convert_queries(queries_data):
         execution_time = snippet['result']['executionTime'] * 100 if snippet['status'] in ('available', 'expired') else -1
         statement = _clean_query(_get_statement(query_data))
         queries.append((original_query_id, execution_time, statement, snippet.get('database', 'default').strip()))
-    except Exception, e:
+    except Exception as e:
       LOG.warning('Skipping upload of %s: %s' % (query_data['uuid'], e))
 
   return queries
@@ -445,7 +446,7 @@ def upload_table_stats(request):
               for col in full_table_stats['columns'][:25]
           ]
 
-        raw_column_stats = [dict([(key, val if val is not None else '') for col_stat in col for key, val in col_stat.iteritems()]) for col in colum_stats]
+        raw_column_stats = [dict([(key, val if val is not None else '') for col_stat in col for key, val in col_stat.items()]) for col in colum_stats]
 
         for col_stats in raw_column_stats:
           column_stats.append({
@@ -461,7 +462,7 @@ def upload_table_stats(request):
             "num_trues": col_stats['num_trues'] if col_stats.get('num_trues', '') != '' else -1,
             "num_falses": col_stats['num_falses'] if col_stats.get('num_falses', '') != '' else -1,
           })
-    except Exception, e:
+    except Exception as e:
       LOG.exception('Skipping upload of %s: %s' % (db_table, e))
 
   api = OptimizerApi(request.user)
@@ -501,7 +502,7 @@ def upload_status(request):
   return JsonResponse(response)
 
 
-class MockRequest():
+class MockRequest(object):
 
   def __init__(self, user, source_platform):
     self.user = user

+ 2 - 0
desktop/libs/metadata/src/metadata/optimizer_api_tests.py

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import zip
+from builtins import object
 import logging
 
 from nose.tools import assert_equal, assert_true, assert_false

+ 2 - 1
desktop/libs/metadata/src/metadata/optimizer_client.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import json
 import logging
 import os
@@ -177,7 +178,7 @@ class OptimizerApi(object):
       status['count'] = len(data)
       return status
 
-    except RestException, e:
+    except RestException as e:
       raise PopupException(e, title=_('Error while accessing Optimizer'))
     finally:
       os.remove(f_queries_path.name)

+ 1 - 0
desktop/libs/metadata/src/metadata/optimizer_client_tests.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import logging
 import time
 

+ 1 - 1
desktop/libs/metadata/src/metadata/prometheus_api.py

@@ -36,7 +36,7 @@ def error_handler(view_fn):
   def decorator(*args, **kwargs):
     try:
       return view_fn(*args, **kwargs)
-    except Exception, e:
+    except Exception as e:
       LOG.exception(e)
       response = {
         'status': -1,

+ 3 - 2
desktop/libs/metadata/src/metadata/prometheus_client.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import logging
 
 from django.core.cache import cache
@@ -60,7 +61,7 @@ class PrometheusApi(object):
       return self._root.get('query', {
         'query': query,
       })['data']
-    except RestException, e:
+    except RestException as e:
       raise PrometheusApiException(e)
 
   def range_query(self, query, start, end, step):
@@ -72,5 +73,5 @@ class PrometheusApi(object):
         'end': end,
         'step': step
       })['data']
-    except RestException, e:
+    except RestException as e:
       raise PrometheusApiException(e)

+ 1 - 1
desktop/libs/metadata/src/metadata/workload_analytics_api.py

@@ -34,7 +34,7 @@ def error_handler(view_fn):
   def decorator(*args, **kwargs):
     try:
       return view_fn(*args, **kwargs)
-    except Exception, e:
+    except Exception as e:
       LOG.exception(e)
       response = {
         'status': -1,

+ 3 - 2
desktop/libs/metadata/src/metadata/workload_analytics_client.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from builtins import object
 import logging
 
 from django.utils.translation import ugettext as _
@@ -25,7 +26,7 @@ from notebook.connectors.altus import _exec
 LOG = logging.getLogger(__name__)
 
 
-class WorkfloadAnalyticsClient():
+class WorkfloadAnalyticsClient(object):
 
   def __init__(self, user):
     self.user = user
@@ -47,7 +48,7 @@ class WorkfloadAnalyticsClient():
 
 
 
-class WorkloadAnalytics():
+class WorkloadAnalytics(object):
 
   def __init__(self, user): pass