Эх сурвалжийг харах

HUE-9198 [assist] Add support for assist file panels where the root path can't be listed

Johan Ahlen 5 жил өмнө
parent
commit
825cabfd56

+ 0 - 1
desktop/core/src/desktop/js/api/apiHelper.js

@@ -737,7 +737,6 @@ class ApiHelper {
    */
   fetchAbfsPath(options) {
     const self = this;
-    options.pathParts.shift();
     let url =
       ABFS_API_PREFIX +
       encodeURI(options.pathParts.join('/')) +

+ 61 - 35
desktop/core/src/desktop/js/ko/components/assist/assistStorageEntry.js

@@ -19,6 +19,7 @@ import * as ko from 'knockout';
 
 import apiHelper from 'api/apiHelper';
 import huePubSub from 'utils/huePubSub';
+import { GET_KNOWN_CONFIG_EVENT } from '../../../utils/hueConfig';
 
 const PAGE_SIZE = 100;
 
@@ -47,17 +48,18 @@ class AssistStorageEntry {
    * @param {object} options.definition
    * @param {string} options.definition.name
    * @param {string} options.definition.type (file, dir)
-   * @param {string} options.type - The storage type ('adls', 'hdfs', 's3')
+   * @param {string} options.source - The storage source
    * @param {string} [options.originalType] - The original storage type ('adl', 's3a')
    * @param {AssistStorageEntry} options.parent
    * @constructor
    */
   constructor(options) {
     const self = this;
-    self.type = options.type;
+    self.source = options.source;
     self.originalType = options.originalType;
     self.definition = options.definition;
     self.parent = options.parent;
+    self.rootPath = options.rootPath || '';
     self.path = '';
     if (self.parent !== null) {
       self.path = self.parent.path;
@@ -102,7 +104,7 @@ class AssistStorageEntry {
   }
 
   dblClick() {
-    huePubSub.publish(TYPE_SPECIFICS[self.type].dblClickPubSubId, this);
+    huePubSub.publish(TYPE_SPECIFICS[self.source.type].dblClickPubSubId, this);
   }
 
   loadPreview() {
@@ -111,7 +113,7 @@ class AssistStorageEntry {
     apiHelper
       .fetchStoragePreview({
         path: self.getHierarchy(),
-        type: self.type,
+        type: self.source.type,
         silenceErrors: true
       })
       .done(data => {
@@ -134,11 +136,12 @@ class AssistStorageEntry {
     self.loading(true);
     self.hasErrors(false);
 
-    apiHelper[TYPE_SPECIFICS[self.type].apiHelperFetchFunction]({
+    apiHelper[TYPE_SPECIFICS[self.source.type].apiHelperFetchFunction]({
       pageSize: PAGE_SIZE,
       page: self.currentPage,
       filter: self.filter().trim() ? self.filter() : undefined,
       pathParts: self.getHierarchy(),
+      rootPath: self.rootPath,
       successCallback: data => {
         self.hasMorePages = data.page.next_page_number > self.currentPage;
         const filteredFiles = data.files.filter(file => file.name !== '.' && file.name !== '..');
@@ -146,7 +149,8 @@ class AssistStorageEntry {
           filteredFiles.map(file => {
             return new AssistStorageEntry({
               originalType: self.originalType,
-              type: self.type,
+              rootPath: self.rootPath,
+              source: self.source,
               definition: file,
               parent: self
             });
@@ -213,10 +217,16 @@ class AssistStorageEntry {
 
   getHierarchy() {
     const self = this;
-    const parts = [];
+    let parts = [];
     let entry = self;
-    while (entry != null) {
-      parts.push(entry.definition.name);
+    while (entry) {
+      if (!entry.parent && entry.definition.name) {
+        const rootParts = entry.definition.name.split('/').filter(Boolean);
+        rootParts.reverse();
+        parts = parts.concat(rootParts);
+      } else {
+        parts.push(entry.definition.name);
+      }
       entry = entry.parent;
     }
     parts.reverse();
@@ -252,7 +262,7 @@ class AssistStorageEntry {
     self.loadingMore(true);
     self.hasErrors(false);
 
-    apiHelper[TYPE_SPECIFICS[self.type].apiHelperFetchFunction]({
+    apiHelper[TYPE_SPECIFICS[self.source.type].apiHelperFetchFunction]({
       pageSize: PAGE_SIZE,
       page: self.currentPage,
       filter: self.filter().trim() ? self.filter() : undefined,
@@ -266,7 +276,8 @@ class AssistStorageEntry {
               file =>
                 new AssistStorageEntry({
                   originalType: self.originalType,
-                  type: self.type,
+                  rootPath: self.rootPath,
+                  source: self.source,
                   definition: file,
                   parent: self
                 })
@@ -338,31 +349,46 @@ class AssistStorageEntry {
     type = type.replace(/adl.*/i, 'adls');
     type = type.replace(/abfs.*/i, 'abfs');
 
-    const rootEntry = new AssistStorageEntry({
-      type: type.toLowerCase(),
-      originalType: typeMatch && typeMatch[1],
-      definition: {
-        name: '/',
-        type: 'dir'
-      },
-      parent: null,
-      apiHelper: apiHelper
-    });
-
-    if (type == 'abfs' || type == 'adls') {
-      // ABFS / ADLS can have domain name in path. To prevent regression with s3 which allow periods in bucket name handle separately.
-      const azureMatch = path.match(
-        /^([^:]+):\/(\/((\w+)@)?[\w]+([\-\.]{1}\w+)*\.[\w]*)?(\/.*)?\/?/i
-      );
-      path = (azureMatch ? azureMatch[6] || '' : path).replace(/(?:^\/)|(?:\/$)/g, '').split('/');
-      if (azureMatch && azureMatch[4]) {
-        path.unshift(azureMatch[4]);
+    huePubSub.publish(GET_KNOWN_CONFIG_EVENT, config => {
+      if (config && config.app_config && config.app_config.browsers) {
+        const source = config.app_config.browsers.interpreters.find(
+          interpreter => interpreter.type === type
+        );
+        if (source) {
+          const rootEntry = new AssistStorageEntry({
+            source: source,
+            originalType: typeMatch && typeMatch[1],
+            definition: {
+              name: '/',
+              type: 'dir'
+            },
+            parent: null,
+            apiHelper: apiHelper
+          });
+
+          if (type === 'abfs' || type === 'adls') {
+            // ABFS / ADLS can have domain name in path. To prevent regression with s3 which allow periods in bucket name handle separately.
+            const azureMatch = path.match(
+              /^([^:]+):\/(\/((\w+)@)?[\w]+([\-\.]{1}\w+)*\.[\w]*)?(\/.*)?\/?/i
+            );
+            path = (azureMatch ? azureMatch[6] || '' : path)
+              .replace(/(?:^\/)|(?:\/$)/g, '')
+              .split('/');
+            if (azureMatch && azureMatch[4]) {
+              path.unshift(azureMatch[4]);
+            }
+          } else {
+            path = (typeMatch ? typeMatch[2] : path).replace(/(?:^\/)|(?:\/$)/g, '').split('/');
+          }
+
+          rootEntry.loadDeep(path, deferred.resolve);
+        } else {
+          deferred.reject();
+        }
+      } else {
+        deferred.reject();
       }
-    } else {
-      path = (typeMatch ? typeMatch[2] : path).replace(/(?:^\/)|(?:\/$)/g, '').split('/');
-    }
-
-    rootEntry.loadDeep(path, deferred.resolve);
+    });
 
     return deferred.promise();
   }

+ 18 - 0
desktop/core/src/desktop/js/ko/components/assist/assistStorageEntry.test.js

@@ -17,6 +17,8 @@
 import $ from 'jquery';
 
 import AssistStorageEntry from './assistStorageEntry';
+import huePubSub from '../../../utils/huePubSub';
+import { GET_KNOWN_CONFIG_EVENT } from '../../../utils/hueConfig';
 
 describe('assistStorageEntry.js', () => {
   it('it should handle domain in ADLS/ABFS', () => {
@@ -57,6 +59,18 @@ describe('assistStorageEntry.js', () => {
       return deferred.promise();
     });
 
+    const pubSpy = jest.spyOn(huePubSub, 'publish').mockImplementation((topic, cb) => {
+      if (topic === GET_KNOWN_CONFIG_EVENT && cb) {
+        cb({
+          app_config: {
+            browsers: {
+              interpreters: [{ type: 'abfs' }]
+            }
+          }
+        });
+      }
+    });
+
     AssistStorageEntry.getEntry('abfs://test.com/path').always(entry => {
       expect(entry.path).toBe('/path');
     });
@@ -69,6 +83,10 @@ describe('assistStorageEntry.js', () => {
     AssistStorageEntry.getEntry('abfs://path@test.com/p2').always(entry => {
       expect(entry.path).toBe('/path/p2');
     });
+    expect(spy).toHaveBeenCalled();
+    expect(pubSpy).toHaveBeenCalled();
+
+    spy.mockRestore();
     spy.mockClear();
   });
 });

+ 6 - 6
desktop/core/src/desktop/js/ko/components/assist/ko.assistPanel.js

@@ -124,13 +124,13 @@ class AssistPanel {
         }
 
         if (self.tabsEnabled) {
-          if (appConfig.browser && appConfig.browser.interpreter_names) {
-            const storageBrowsers = appConfig.browser.interpreter_names.filter(
+          if (appConfig.browser && appConfig.browser.interpreters) {
+            const storageBrowsers = appConfig.browser.interpreters.filter(
               interpreter =>
-                interpreter === 'adls' ||
-                interpreter === 'hdfs' ||
-                interpreter === 's3' ||
-                interpreter === 'abfs'
+                interpreter.type === 'adls' ||
+                interpreter.type === 'hdfs' ||
+                interpreter.type === 's3' ||
+                interpreter.type === 'abfs'
             );
 
             if (storageBrowsers.length) {

+ 46 - 20
desktop/core/src/desktop/js/ko/components/assist/ko.assistStoragePanel.js

@@ -45,11 +45,11 @@ const TEMPLATE = `
 
   <script type="text/html" id="assist-storage-header-actions">
     <div class="assist-db-header-actions">
-      <!-- ko if: type !== 's3' && type !== 'abfs' -->
+      <!-- ko if: source.type !== 's3' && source.type !== 'abfs' -->
       <a class="inactive-action" href="javascript:void(0)" data-bind="click: goHome, attr: { title: I18n('Go to ' + window.USER_HOME_DIR) }"><i class="pointer fa fa-home"></i></a>
       <!-- ko if: window.SHOW_UPLOAD_BUTTON -->
       <a class="inactive-action" data-bind="dropzone: {
-            url: '/filebrowser/upload/file?dest=' + (type === 'adls' ? 'adl:' : '') + path,
+            url: '/filebrowser/upload/file?dest=' + (source.type === 'adls' ? 'adl:' : '') + path,
             params: { dest: path },
             paramName: 'hdfs_file',
             onError: function(x, e){ $(document).trigger('error', e); },
@@ -62,7 +62,7 @@ const TEMPLATE = `
       </a>
       <!-- /ko -->
       <!-- /ko -->
-      <!-- ko if: type === 'abfs' && path !== '/' && window.SHOW_UPLOAD_BUTTON -->
+      <!-- ko if: source.type === 'abfs' && path !== '/' && window.SHOW_UPLOAD_BUTTON -->
       <a class="inactive-action" data-bind="dropzone: {
             url: '/filebrowser/upload/file?dest=' + 'abfs:/' + path,
             params: { dest: 'abfs:/' + path },
@@ -93,7 +93,7 @@ const TEMPLATE = `
   <div class="assist-flex-fill">
     <ul class="assist-tables" data-bind="foreach: sources">
       <li class="assist-table">
-        <a class="assist-table-link" href="javascript: void(0);" data-bind="click: function () { $parent.activeSource($data); }"><i class="fa fa-fw fa-server muted valign-middle"></i> <span data-bind="text: $data.toUpperCase()"></span></a>
+        <a class="assist-table-link" href="javascript: void(0);" data-bind="click: function () { $parent.activeSource($data); }"><i class="fa fa-fw fa-server muted valign-middle"></i> <span data-bind="text: $data.displayName"></span></a>
       </li>
     </ul>
   </div>
@@ -118,7 +118,7 @@ const TEMPLATE = `
     <a href="javascript: void(0);" data-bind="click: function () { $parent.activeSource(undefined) }">
       <i class="fa fa-fw fa-chevron-left"></i>
       <i class="fa fa-fw fa-server"></i>
-      <span data-bind="text: $parent.activeSource().toUpperCase()"></span>
+      <span data-bind="text: $parent.activeSource().displayName"></span>
     </a>
     <!-- /ko -->
     <!-- ko template: 'assist-storage-header-actions' --><!-- /ko -->
@@ -152,7 +152,7 @@ const TEMPLATE = `
             <!-- ko if: definition.type === 'file' -->
             <i class="fa fa-fw fa-file-o muted valign-middle"></i>
             <!-- /ko -->
-            <span draggable="true" data-bind="text: definition.name, draggableText: { text: '\\'' + path + '\\'', meta: {'type': type, 'definition': definition} }"></span>
+            <span draggable="true" data-bind="text: definition.name, draggableText: { text: '\\'' + path + '\\'', meta: {'type': source.type, 'definition': definition} }"></span>
           </a>
         </li>
       </ul>
@@ -174,30 +174,49 @@ const TEMPLATE = `
   <!-- /ko -->
 `;
 
+const rootPathRegex = /.*%3A%2F%2F(.+)$/;
+
+/**
+ * This takes the initial path from the "browser" config, used in cases where the users can't access '/'
+ */
+const getRootPath = source => {
+  if (source) {
+    const match = source.page.match(rootPathRegex);
+    if (match) {
+      return match[1] + '/';
+    }
+  }
+  return '';
+};
+
 class AssistStoragePanel {
   /**
    * @param {Object} options
-   * @param {String[]} options.sources
+   * @param {Interpreter[]} options.sources
    * @constructor
    **/
   constructor(options) {
     this.sources = ko.observableArray(options.sources);
 
-    let lastSource = apiHelper.getFromTotalStorage('assist', 'lastStorageSource', 'hdfs');
+    const lastSourceType = apiHelper.getFromTotalStorage('assist', 'lastStorageSource', 'hdfs');
+
+    let foundLastSource = this.sources().find(source => source.type === lastSourceType);
 
-    if (options.sources.indexOf(lastSource) === -1) {
-      lastSource = options.sources.indexOf('hdfs') !== -1 ? 'hdfs' : options.sources[0];
+    if (!foundLastSource && this.sources().length) {
+      foundLastSource = this.sources().find(source => source.type === 'hdfs') || this.sources()[0];
     }
 
-    this.activeSource = ko.observable(lastSource);
+    this.activeSource = ko.observable(foundLastSource);
     this.loading = ko.observable();
     this.initialized = false;
+    this.rootPath = getRootPath(this.activeSource());
 
     this.selectedStorageEntry = ko.observable();
 
     this.activeSource.subscribe(newValue => {
       if (newValue) {
-        apiHelper.setInTotalStorage('assist', 'lastStorageSource', newValue);
+        this.rootPath = getRootPath(this.activeSource());
+        apiHelper.setInTotalStorage('assist', 'lastStorageSource', newValue.type);
         this.selectedStorageEntry(undefined);
         this.reload();
       }
@@ -209,15 +228,17 @@ class AssistStoragePanel {
     });
 
     huePubSub.subscribe('assist.storage.refresh', () => {
-      apiHelper.clearStorageCache(this.activeSource());
+      apiHelper.clearStorageCache(this.activeSource().type);
       this.reload();
     });
 
     huePubSub.subscribe('assist.storage.go.home', () => {
       const path =
-        this.activeSource() === 's3' || this.activeSource() === 'abfs' ? '/' : window.USER_HOME_DIR;
+        this.activeSource().type === 's3' || this.activeSource().type === 'abfs'
+          ? '/'
+          : window.USER_HOME_DIR;
       this.loadPath(path);
-      apiHelper.setInTotalStorage('assist', 'currentStoragePath_' + this.activeSource(), path);
+      apiHelper.setInTotalStorage('assist', 'currentStoragePath_' + this.activeSource().type, path);
     });
 
     this.init();
@@ -225,13 +246,18 @@ class AssistStoragePanel {
 
   loadPath(path) {
     this.loading(true);
-    const parts = path.split('/');
+    let relativePath = path;
+    if (this.rootPath) {
+      relativePath = relativePath.replace(this.rootPath, '/');
+    }
+    const parts = relativePath.split('/');
     parts.shift();
 
     const currentEntry = new AssistStorageEntry({
-      type: this.activeSource(),
+      source: this.activeSource(),
+      rootPath: this.rootPath,
       definition: {
-        name: '/',
+        name: this.rootPath,
         type: 'dir'
       },
       parent: null
@@ -248,8 +274,8 @@ class AssistStoragePanel {
     this.loadPath(
       apiHelper.getFromTotalStorage(
         'assist',
-        'currentStoragePath_' + this.activeSource(),
-        this.activeSource() === 'hdfs' ? window.USER_HOME_DIR : '/'
+        'currentStoragePath_' + this.activeSource().type,
+        this.activeSource().type === 'hdfs' ? window.USER_HOME_DIR : '/'
       )
     );
   }