Эх сурвалжийг харах

HUE-8161 [metadata] Automatically create a namespace to enrich the catalog

Romain Rigaux 7 жил өмнө
parent
commit
5800ddc

+ 47 - 0
desktop/libs/metadata/src/metadata/navigator_client.py

@@ -43,6 +43,8 @@ VERSION = 'v9'
 _JSON_CONTENT_TYPE = 'application/json'
 CLUSTER_SOURCE_IDS_CACHE_KEY = 'nav-cluster-source-ids-id'
 
+_HAS_CATALOG_NAMESPACE = None
+
 
 def get_cluster_source_ids(api):
   '''
@@ -114,6 +116,7 @@ class NavigatorApi(object):
   http://cloudera.github.io/navigator/apidocs/v3/index.html
   """
   DEFAULT_SEARCH_FIELDS = (('originalName', 3), ('originalDescription', 1), ('name', 10), ('description', 3), ('tags', 5))
+  CATALOG_NAMESPACE = '__cloudera_internal_catalog_hue'
 
   def __init__(self, user=None):
     self._api_url = '%s/%s' % (NAVIGATOR.API_URL.get().strip('/'), VERSION)
@@ -129,6 +132,8 @@ class NavigatorApi(object):
     self.__headers = {}
     self.__params = ()
 
+    self._fillup_properties()
+
 
   def _get_types_from_sources(self, sources):
     default_entity_types = entity_types = ('DATABASE', 'TABLE', 'PARTITION', 'FIELD', 'FILE', 'VIEW', 'S3BUCKET', 'OPERATION', 'DIRECTORY')
@@ -532,6 +537,16 @@ class NavigatorApi(object):
       LOG.error(msg)
       raise NavigatorApiException(e.message)
 
+
+  def get_namespace_properties(self, namespace):
+    try:
+      return self._root.get('models/namespaces/%(namespace)s/properties' % {'namespace': namespace})
+    except RestException, e:
+      msg = 'Failed to create namespace %s property' % namespace
+      LOG.error(msg)
+      raise NavigatorApiException(e.message)
+
+
   def map_namespace_property(self, clazz, properties):
     try:
       data = json.dumps(properties)
@@ -542,6 +557,38 @@ class NavigatorApi(object):
       raise NavigatorApiException(e.message)
 
 
+  def _fillup_properties(self):
+    global _HAS_CATALOG_NAMESPACE
+
+    if _HAS_CATALOG_NAMESPACE is None:
+      response = self.get_namespace(namespace=NavigatorApi.CATALOG_NAMESPACE)
+      if not response:
+        self.create_namespace(namespace=NavigatorApi.CATALOG_NAMESPACE, description="Set of fields to augment the data catalog")
+
+      properties = self.get_namespace_properties(namespace=NavigatorApi.CATALOG_NAMESPACE)
+
+      if not [_property for _property in properties if _property['name'] == 'relatedQueries']:
+        self.create_namespace_property(namespace=NavigatorApi.CATALOG_NAMESPACE, properties={
+          "name": "relatedQueries",
+          "displayName": "Related queries",
+          "description": "List of saved SQL queries UUIDs that show how to analyze this table",
+          "multiValued": True,
+          "maxLength": 36,
+          "pattern": ".*", # UUID
+          "enumValues": None,
+          "type": "TEXT"
+         })
+
+        # Might want to check if the mapping is already done
+        for clazz in ('hv_table', 'hv_view'):
+          self.map_namespace_property(clazz, properties=[{
+             "namespace": NavigatorApi.CATALOG_NAMESPACE,
+             "name": "relatedQueries"
+          }])
+
+      _HAS_CATALOG_NAMESPACE = True
+
+
   def _get_boosted_term(self, term):
     return 'AND'.join([
       '(%s)' % 'OR'.join(['(%s:*%s*^%s)' % (field, term, weight) for (field, weight) in NavigatorApi.DEFAULT_SEARCH_FIELDS]),  # Matching fields