navigator.py 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160
  1. #!/usr/bin/env python
  2. # -- coding: utf-8 --
  3. # Licensed to Cloudera, Inc. under one
  4. # or more contributor license agreements. See the NOTICE file
  5. # distributed with this work for additional information
  6. # regarding copyright ownership. Cloudera, Inc. licenses this file
  7. # to you under the Apache License, Version 2.0 (the
  8. # "License"); you may not use this file except in compliance
  9. # with the License. You may obtain a copy of the License at
  10. #
  11. # http://www.apache.org/licenses/LICENSE-2.0
  12. #
  13. # Unless required by applicable law or agreed to in writing, software
  14. # distributed under the License is distributed on an "AS IS" BASIS,
  15. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16. # See the License for the specific language governing permissions and
  17. # limitations under the License.
  18. import json
  19. import logging
  20. from desktop.lib.rest.http_client import HttpClient, RestException
  21. from desktop.lib.rest import resource
  22. from metadata.conf import NAVIGATOR
  23. LOG = logging.getLogger(__name__)
  24. class NavigatorApiException(Exception):
  25. pass
  26. class NavigatorApi(object):
  27. """
  28. http://cloudera.github.io/navigator/apidocs/v2/index.html
  29. """
  30. def __init__(self, api_url=None, user=None, password=None):
  31. self._api_url = (api_url or NAVIGATOR.API_URL.get()).strip('/')
  32. self._username = user or NAVIGATOR.AUTH_USERNAME.get()
  33. self._password = password or NAVIGATOR.AUTH_PASSWORD.get()
  34. self._client = HttpClient(self._api_url, logger=LOG)
  35. self._client.set_basic_auth(self._username, self._password)
  36. self._root = resource.Resource(self._client)
  37. self.__headers = {}
  38. self.__params = ()
  39. def find_entity(self, source_type, type, name, **filters):
  40. """
  41. GET /api/v2/entities?query=((sourceType:<source_type>)AND(type:<type>)AND(originalName:<name>))
  42. http://cloudera.github.io/navigator/apidocs/v2/path__v2_entities.html
  43. """
  44. try:
  45. params = self.__params
  46. query_filters = {
  47. 'sourceType': source_type,
  48. 'type': type,
  49. 'originalName': name,
  50. 'deleted': 'false'
  51. }
  52. for key, value in filters.items():
  53. query_filters[key] = value
  54. filter_query = 'AND'.join('(%s:%s)' % (key, value) for key, value in query_filters.items())
  55. params += (
  56. ('query', filter_query),
  57. ('offset', 0),
  58. ('limit', 2), # We are looking for single entity, so limit to 2 to check for multiple results
  59. )
  60. response = self._root.get('entities', headers=self.__headers, params=params)
  61. if not response:
  62. raise NavigatorApiException('Could not find entity with query filters: %s' % str(query_filters))
  63. elif len(response) > 1:
  64. raise NavigatorApiException('Found more than 1 entity with query filters: %s' % str(query_filters))
  65. return response[0]
  66. except RestException, e:
  67. msg = 'Failed to find entity: %s' % str(e)
  68. LOG.exception(msg)
  69. raise NavigatorApiException(msg)
  70. def get_entity(self, entity_id):
  71. """
  72. GET /api/v2/entities/:id
  73. http://cloudera.github.io/navigator/apidocs/v2/path__v2_entities_-id-.html
  74. """
  75. try:
  76. return self._root.get('entities/%s' % entity_id, headers=self.__headers, params=self.__params)
  77. except RestException, e:
  78. msg = 'Failed to get entity %s: %s' % (entity_id, str(e))
  79. LOG.exception(msg)
  80. raise NavigatorApiException(msg)
  81. def update_entity(self, entity_id, **metadata):
  82. """
  83. PUT /api/v2/entities/:id
  84. http://cloudera.github.io/navigator/apidocs/v2/path__v2_entities_-id-.html
  85. """
  86. try:
  87. # TODO: Check permissions of entity
  88. data = json.dumps(metadata)
  89. return self._root.put('entities/%s' % entity_id, params=self.__params, data=data)
  90. except RestException, e:
  91. msg = 'Failed to update entity %s: %s' % (entity_id, str(e))
  92. LOG.exception(msg)
  93. raise NavigatorApiException(msg)
  94. def get_database(self, name):
  95. return self.find_entity(source_type='HIVE', type='DATABASE', name=name)
  96. def get_table(self, database_name, table_name):
  97. parent_path = '\/%s' % database_name
  98. return self.find_entity(source_type='HIVE', type='TABLE', name=table_name, parentPath=parent_path)
  99. def get_directory(self, path):
  100. dir_name, dir_path = self._clean_path(path)
  101. return self.find_entity(source_type='HDFS', type='DIRECTORY', name=dir_name, fileSystemPath=dir_path)
  102. def get_file(self, path):
  103. file_name, file_path = self._clean_path(path)
  104. return self.find_entity(source_type='HDFS', type='FILE', name=file_name, fileSystemPath=file_path)
  105. def add_tags(self, entity_id, tags):
  106. entity = self.get_entity(entity_id)
  107. new_tags = entity['tags'] or []
  108. new_tags.extend(tags)
  109. return self.update_entity(entity_id, tags=new_tags)
  110. def delete_tags(self, entity_id, tags):
  111. entity = self.get_entity(entity_id)
  112. new_tags = entity['tags'] or []
  113. for tag in tags:
  114. if tag in new_tags:
  115. new_tags.remove(tag)
  116. return self.update_entity(entity_id, tags=new_tags)
  117. def _clean_path(self, path):
  118. return path.rstrip('/').split('/')[-1], self._escape_slashes(path.rstrip('/'))
  119. def _escape_slashes(self, s):
  120. return s.replace('/', '\/')