Explorar o código

HUE-5655 [metadata] Add support for S3 sources

Romain Rigaux %!s(int64=9) %!d(string=hai) anos
pai
achega
d54e93b0dd

+ 18 - 5
desktop/core/src/desktop/templates/assist.mako

@@ -1248,7 +1248,7 @@ from notebook.conf import ENABLE_QUERY_BUILDER
             <!-- ko if: typeof click !== 'undefined' -->
             <a class="pointer" data-bind="click: click, html: hue_name" target="_blank" ></a>
             <!-- /ko -->
-            <!-- ko if: typeof click === 'undefined' -->
+            <!-- ko if: typeof click === 'undefined' && typeof link !== 'undefined'-->
             <a class="pointer" data-bind="attr: { 'href': link }, text: originalName" target="_blank" ></a>
             <!-- /ko -->
             <div class="doc-desc" data-bind="html: hue_description"></div>
@@ -1711,6 +1711,7 @@ from notebook.conf import ENABLE_QUERY_BUILDER
         'OPERATION_EXECUTION': 'fa-cog',
         'DIRECTORY': 'fa-folder-o',
         'FILE': 'fa-file-o',
+        'S3BUCKET': 'fa-cubes',
         'SUB_OPERATION': 'fa-code-fork',
         'COLLECTION': 'fa-search',
         'HBASE': 'fa-th-large',
@@ -1796,8 +1797,7 @@ from notebook.conf import ENABLE_QUERY_BUILDER
               name: '${ _("S3") }',
               type: 's3',
               icon: 'fa-cubes',
-              minHeight: 50,
-              showNavSearch: false
+              minHeight: 50
             }));
           }
 
@@ -1943,12 +1943,25 @@ from notebook.conf import ENABLE_QUERY_BUILDER
                 }
                 case 'DIRECTORY': {
                   entity.originalDescription = entity.parentPath;
-                  entity.link = '/filebrowser/#' + entity.fileSystemPath;
+                  if (entity.sourceType == 'S3') {
+                    entity.link = '/filebrowser/view=S3A://#s3a://' + entity.bucketName + '/' + entity.fileSystemPath;
+                  } else {
+                    entity.link = '/filebrowser/#' + entity.fileSystemPath;
+                  }
                   break;
                 }
                 case 'FILE': {
                   entity.originalDescription = entity.parentPath;
-                  entity.link = '/filebrowser/#' + entity.fileSystemPath;
+                  if (entity.sourceType == 'S3') {
+                    entity.link = '/filebrowser/view=S3A://#s3a://' + entity.bucketName + '/' + entity.fileSystemPath;
+                  } else {
+                    entity.link = '/filebrowser/#' + entity.fileSystemPath;
+                  }
+                  break;
+                }
+                case 'S3BUCKET': {
+                  entity.originalDescription = '${ _("Region") }: ' + entity.region;
+                  entity.link = '/filebrowser/view=S3A://#s3a://' + entity.originalName;
                   break;
                 }
                 case 'SUB_OPERATION': {

+ 1 - 1
desktop/libs/metadata/src/metadata/conf.py

@@ -163,7 +163,7 @@ NAVIGATOR = ConfigSection(
     APPLY_SENTRY_PERMISSIONS = Config(
       key="apply_sentry_permissions",
       help=_t("Perform Sentry privilege filtering."),
-      default=True,
+      default=False,
       type=coerce_bool
     ),
   )

+ 4 - 1
desktop/libs/metadata/src/metadata/navigator_api.py

@@ -244,7 +244,10 @@ def _augment_highlighting(query_s, records):
   for record in records:
     name = record.get('originalName', '')
     record['hue_description'] = ''
-    record['hue_name'] = (record.get('parentPath', '').replace('/', '.') + '.').lstrip('.') if record.get('parentPath') else ''
+    record['hue_name'] = record.get('parentPath', '') if record.get('parentPath') else ''
+
+    if record['hue_name'] and record.get('sourceType', '') != 'S3':
+      record['hue_name'] = (record['hue_name'].replace('/', '.') + '.').lstrip('.')
 
     for term in ts:
       name = _highlight(term, name)

+ 14 - 2
desktop/libs/metadata/src/metadata/navigator_client.py

@@ -67,14 +67,17 @@ class NavigatorApi(object):
 
 
   def _get_types_from_sources(self, sources):
-    default_entity_types = entity_types = ('DATABASE', 'TABLE', 'PARTITION', 'FIELD', 'FILE', 'VIEW', 'OPERATION', 'DIRECTORY')
+    default_entity_types = entity_types = ('DATABASE', 'TABLE', 'PARTITION', 'FIELD', 'FILE', 'VIEW', 'S3BUCKET', 'OPERATION', 'DIRECTORY')
 
     if 'sql' in sources or 'hive' in sources or 'impala' in sources:
-      default_entity_types = ('TABLE', 'VIEW')
       entity_types = ('TABLE', 'VIEW', 'DATABASE', 'PARTITION', 'FIELD')
+      default_entity_types = ('TABLE', 'VIEW')
     elif 'hdfs' in sources:
       entity_types = ('FILE', 'DIRECTORY')
       default_entity_types  = ('FILE', 'DIRECTORY')
+    elif 's3' in sources:
+      entity_types = ('FILE', 'DIRECTORY', 'S3BUCKET')
+      default_entity_types  = ('DIRECTORY', 'S3BUCKET')
 
     return default_entity_types, entity_types
 
@@ -99,6 +102,8 @@ class NavigatorApi(object):
 
       query_clauses = []
       user_filters = []
+      source_type_filter = []
+
       for term in search_terms:
         if ':' not in term:
           query_clauses.append('OR'.join(['(%s:*%s*)' % (field, term) for field in search_fields]))
@@ -117,8 +122,12 @@ class NavigatorApi(object):
 
       user_filter_clause = 'OR '.join(['(%s)' % f for f in user_filters]) or '*'
       source_filter_clause = 'OR'.join(['(%s:%s)' % ('type', entity_type) for entity_type in default_entity_types])
+      if 's3' in sources:
+        source_type_filter.append('sourceType:s3')
 
       filter_query = '%s AND (%s) AND (%s)' % (filter_query, user_filter_clause, source_filter_clause)
+      if source_type_filter:
+        filter_query += ' AND (%s)' % 'OR '.join(source_type_filter)
 
       params += (
         ('query', filter_query),
@@ -157,6 +166,9 @@ class NavigatorApi(object):
           fq_type = default_entity_types
         elif 'hdfs' in sources:
           fq_type = entity_types
+        elif 's3' in sources:
+          fq_type = default_entity_types
+          filterQueries.append('sourceType:s3')
 
         if query_s.strip().endswith('type:*'): # To list all available types
           fq_type = entity_types