Ver código fonte

[metadata] Initial metadata API

POST /metadata/api/navigator/find_entity
  'type': database, table, directory, file
  'name': required for database or table
  'database': required for table
  'path': required for directory or file

GET /metadata/api/navigator/get_entity
  'id': entity_id

POST /metadata/api/navigator/add_tags
  'id': entity_id
  'tags': list of tags to add to entity metadata

POST /metadata/api/navigator/delete_tags
  'id': entity_id
  'tags': list of tags to add to entity metadata

POST /metadata/api/navigator/update_properties
  'id': entity_id
  'properties': map of key-value properties to add/update for entity

POST /metadata/api/navigator/delete_properties
  'id': entity_id
  'properties': list of property keys to delete

All responses include a status (0 if success) and entity object where the 'identifier' is the entity_id
Jenny Kim 10 anos atrás
pai
commit
e8b14a7

+ 1 - 1
desktop/conf.dist/hue.ini

@@ -1356,7 +1356,7 @@
 
   [[navigator]]
   # Navigator API URL with version
-  ## api_url=http://jennykim-1.vpc.cloudera.com:7187/api/v2
+  ## api_url=http://localhost:7187/api/v2
 
   # Navigator API HTTP authentication username and password
   # Override the desktop default username and password of the hue user used for authentications with other services.

+ 1 - 1
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1358,7 +1358,7 @@
 
   [[navigator]]
   # Navigator API URL with version
-  ## api_url=http://jennykim-1.vpc.cloudera.com:7187/api/v2
+  ## api_url=http://localhost:7187/api/v2
 
   # Navigator API HTTP authentication username and password
   # Override the desktop default username and password of the hue user used for authentications with other services.

+ 0 - 0
desktop/libs/metadata/__init__.py


+ 2 - 2
desktop/libs/metadata/setup.py

@@ -21,11 +21,11 @@ setup(
       name = "metadata",
       version = VERSION,
       url = 'http://github.com/cloudera/hue',
-      description = "Search Libraries",
+      description = "Metadata Libraries",
       packages = find_packages('src'),
       package_dir = {'': 'src' },
       install_requires = ['setuptools', 'desktop'],
       # Even libraries need to be registered as desktop_apps,
       # if they have configuration, like this one.
-      entry_points = { 'desktop.sdk.lib': 'metadata=metadata' },
+      entry_points = { 'desktop.sdk.application': 'metadata=metadata' },
 )

+ 4 - 0
desktop/libs/metadata/src/metadata/navigator.py

@@ -28,6 +28,10 @@ from metadata.conf import NAVIGATOR
 LOG = logging.getLogger(__name__)
 
 
+def is_navigator_enabled():
+  return NAVIGATOR.API_URL.get()
+
+
 class NavigatorApiException(Exception):
   pass
 

+ 180 - 0
desktop/libs/metadata/src/metadata/navigator_api.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+
+from django.http import Http404
+from django.utils.translation import ugettext as _
+from django.views.decorators.http import require_POST
+
+from desktop.lib.django_util import JsonResponse
+from desktop.lib.i18n import force_unicode
+
+from metadata.navigator import NavigatorApi, is_navigator_enabled
+
+LOG = logging.getLogger(__name__)
+
+
+class MetadataApiException(Exception):
+  pass
+
+
+def error_handler(view_fn):
+  def decorator(*args, **kwargs):
+    try:
+      if is_navigator_enabled():
+        return view_fn(*args, **kwargs)
+      else:
+        raise MetadataApiException('Navigator API is not configured.')
+    except Http404, e:
+      raise e
+    except Exception, e:
+      LOG.exception(str(e))
+      response = {
+        'status': -1,
+        'message': force_unicode(str(e))
+      }
+    return JsonResponse(response, status=500)
+  return decorator
+
+
+@require_POST
+@error_handler
+def find_entity(request):
+  response = {'status': -1}
+
+  api = NavigatorApi()
+  entity_type = json.loads(request.POST.get('type', ''))
+
+  if not entity_type:
+    raise MetadataApiException("find_entity requires a type value, e.g. - 'database', 'table', 'file'")
+
+  if entity_type.lower() == 'database':
+    name = json.loads(request.POST.get('name', ''))
+    if not name:
+      raise MetadataApiException('get_database requires name param')
+    response['entity'] = api.get_database(name)
+  elif entity_type.lower() == 'table':
+    database = json.loads(request.POST.get('database', ''))
+    name = request.POST.get('name', '')
+    if not database or not name:
+      raise MetadataApiException('get_table requires database and name param')
+    response['entity'] = api.get_table(database, name)
+  elif entity_type.lower() == 'directory':
+    path = json.loads(request.POST.get('path', ''))
+    if not path:
+      raise MetadataApiException('get_directory requires path param')
+    response['entity'] = api.get_directory(path)
+  elif entity_type.lower() == 'file':
+    path = json.loads(request.POST.get('path', ''))
+    if not path:
+      raise MetadataApiException('get_file requires path param')
+    response['entity'] = api.get_file(path)
+  else:
+    raise MetadataApiException("type %s is unrecognized" % entity_type)
+
+  response['status'] = 0
+  return JsonResponse(response)
+
+
+@error_handler
+def get_entity(request):
+  response = {'status': -1}
+
+  api = NavigatorApi()
+  entity_id = request.REQUEST.get('id')
+
+  if not entity_id:
+    raise MetadataApiException("get_entity requires an 'id' parameter")
+
+  response['entity'] = api.get_entity(entity_id)
+  response['status'] = 0
+
+  return JsonResponse(response)
+
+
+@require_POST
+@error_handler
+def add_tags(request):
+  response = {'status': -1}
+
+  api = NavigatorApi()
+  entity_id = json.loads(request.POST.get('id', ''))
+  tags = json.loads(request.POST.get('tags', []))
+
+  if not entity_id or not tags or not isinstance(tags, list):
+    response['error'] = _("add_tags requires an 'id' parameter and 'tags' parameter that is a non-empty list of tags")
+  else:
+    response['entity'] = api.add_tags(entity_id, tags)
+    response['status'] = 0
+
+  return JsonResponse(response)
+
+
+@require_POST
+@error_handler
+def delete_tags(request):
+  response = {'status': -1}
+
+  api = NavigatorApi()
+  entity_id = json.loads(request.POST.get('id', ''))
+  tags = json.loads(request.POST.get('tags', []))
+
+  if not entity_id or not tags or not isinstance(tags, list):
+    response['error'] = _("add_tags requires an 'id' parameter and 'tags' parameter that is a non-empty list of tags")
+  else:
+    response['entity'] = api.delete_tags(entity_id, tags)
+    response['status'] = 0
+
+  return JsonResponse(response)
+
+
+@require_POST
+@error_handler
+def update_properties(request):
+  response = {'status': -1}
+
+  api = NavigatorApi()
+  entity_id = json.loads(request.POST.get('id', ''))
+  properties = json.loads(request.POST.get('properties', {}))
+
+  if not entity_id or not properties or not isinstance(properties, dict):
+    response['error'] = _("update_properties requires an 'id' parameter and 'properties' parameter that is a non-empty dict")
+  else:
+    response['entity'] = api.update_properties(entity_id, properties)
+    response['status'] = 0
+
+  return JsonResponse(response)
+
+
+@require_POST
+@error_handler
+def delete_properties(request):
+  response = {'status': -1}
+
+  api = NavigatorApi()
+  entity_id = json.loads(request.POST.get('id', ''))
+  keys = json.loads(request.POST.get('keys', []))
+
+  if not entity_id or not keys or not isinstance(keys, list):
+    response['error'] = _("update_properties requires an 'id' parameter and 'keys' parameter that is a non-empty list")
+  else:
+    response['entity'] = api.delete_properties(entity_id, keys)
+    response['status'] = 0
+
+  return JsonResponse(response)

+ 22 - 0
desktop/libs/metadata/src/metadata/settings.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DJANGO_APPS = [ "metadata" ]
+NICE_NAME = "Metadata"
+REQUIRES_HADOOP = False
+
+IS_URL_NAMESPACED = True

+ 42 - 29
desktop/libs/metadata/src/metadata/tests.py

@@ -28,7 +28,7 @@ from hadoop.pseudo_hdfs4 import is_live_cluster
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import add_to_group, grant_access
 
-from metadata.navigator import NavigatorApi
+from metadata.navigator import NavigatorApi, is_navigator_enabled
 
 
 LOG = logging.getLogger(__name__)
@@ -36,24 +36,10 @@ LOG = logging.getLogger(__name__)
 
 class TestNavigatorApi(object):
 
-  @staticmethod
-  def is_navigator_enabled():
-    is_enabled = True
-    try:
-      from metadata.conf import NAVIGATOR
-      if not NAVIGATOR.API_URL.get():
-        is_enabled = False
-    except:
-      LOG.info('Testing navigator requires a configured navigator api_url')
-      is_enabled = False
-
-    return is_enabled
-
-
   @classmethod
   def setup_class(cls):
 
-    if not is_live_cluster() or not cls.is_navigator_enabled():
+    if not is_live_cluster() or not is_navigator_enabled():
       raise SkipTest
 
     cls.client = make_logged_in_client(username='test', is_superuser=False)
@@ -76,27 +62,54 @@ class TestNavigatorApi(object):
     assert_true('identity' in entity, entity)
 
 
-  def test_update_tags(self):
+  def test_api_find_entity(self):
+    resp = self.client.post(reverse('metadata:find_entity'), self._format_json_body({'type': 'database', 'name': 'default'}))
+    json_resp = json.loads(resp.content)
+    assert_equal(0, json_resp['status'])
+    assert_true('entity' in json_resp, json_resp)
+    assert_true('identity' in json_resp['entity'], json_resp)
+
+
+  def test_api_tags(self):
     entity = self.api.find_entity(source_type='HIVE', type='DATABASE', name='default')
+    entity_id = entity['identity']
     tags = entity['tags'] or []
 
-    entity = self.api.add_tags(entity['identity'], ['hue_test'])
-    assert_equal(tags + ['hue_test'], entity['tags'])
+    resp = self.client.post(reverse('metadata:add_tags'), self._format_json_body({'id': entity_id}))
+    json_resp = json.loads(resp.content)
+    # add_tags requires a list of tags
+    assert_equal(-1, json_resp['status'])
+
+    resp = self.client.post(reverse('metadata:add_tags'), self._format_json_body({'id': entity_id, 'tags': ['hue_test']}))
+    json_resp = json.loads(resp.content)
+    assert_equal(0, json_resp['status'], json_resp)
+    assert_equal(tags + ['hue_test'], json_resp['entity']['tags'])
 
-    entity = self.api.delete_tags(entity['identity'], ['hue_test'])
-    new_tags = entity['tags'] or []
-    assert_equal(tags, new_tags)
+    resp = self.client.post(reverse('metadata:delete_tags'), self._format_json_body({'id': entity_id, 'tags': ['hue_test']}))
+    json_resp = json.loads(resp.content)
+    assert_equal(0, json_resp['status'], json_resp)
+    assert_equal(entity['tags'] , json_resp['entity']['tags'])
 
 
-  def test_update_properties(self):
+  def test_api_properties(self):
     entity = self.api.find_entity(source_type='HIVE', type='DATABASE', name='default')
+    entity_id = entity['identity']
     props = entity['properties'] or {}
 
-    entity = self.api.update_properties(entity['identity'], {'hue': 'test'})
+    resp = self.client.post(reverse('metadata:update_properties'), self._format_json_body({'id': entity_id, 'properties': {'hue': 'test'}}))
+    json_resp = json.loads(resp.content)
+    assert_equal(0, json_resp['status'], json_resp)
     props.update({'hue': 'test'})
-    assert_equal(props, entity['properties'])
+    assert_equal(props, json_resp['entity']['properties'])
+
+    resp = self.client.post(reverse('metadata:delete_properties'), self._format_json_body({'id': entity_id, 'keys': ['hue']}))
+    json_resp = json.loads(resp.content)
+    assert_equal(0, json_resp['status'], json_resp)
+    assert_equal(entity['properties'], json_resp['entity']['properties'])
+
 
-    entity = self.api.delete_properties(entity['identity'], ['hue'])
-    del props['hue']
-    new_props = entity['properties'] or {}
-    assert_equal(props, new_props)
+  def _format_json_body(self, post_dict):
+    json_dict = {}
+    for key, value in post_dict.items():
+      json_dict[key] = json.dumps(value)
+    return json_dict

+ 28 - 0
desktop/libs/metadata/src/metadata/urls.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from django.conf.urls import patterns, url
+
+# Navigator API
+urlpatterns = patterns('metadata.navigator_api',
+  url(r'^api/navigator/find_entity/?$', 'find_entity', name='find_entity'),
+  url(r'^api/navigator/get_entity/?$', 'get_entity', name='get_entity'),
+  url(r'^api/navigator/add_tags/?$', 'add_tags', name='add_tags'),
+  url(r'^api/navigator/delete_tags/?$', 'delete_tags', name='delete_tags'),
+  url(r'^api/navigator/update_properties/?$', 'update_properties', name='update_properties'),
+  url(r'^api/navigator/delete_properties/?$', 'delete_properties', name='delete_properties'),
+)