Browse Source

[ui-storagebrowser] adds and reads configs from .ini files (#3922)

* [ui-storagebrowser] adds and reads configs from .ini files

* removes show download button from global constants

* moves the storage browser config to separate group

* adds storage_browser key in config
Ram Prasad Agarwal 11 months ago
parent
commit
9c6991045c

+ 9 - 4
desktop/core/src/desktop/api2.py

@@ -78,8 +78,9 @@ from desktop.models import (
   uuid_default,
 )
 from desktop.views import get_banner_message, serve_403_error
-from filebrowser.conf import RESTRICT_FILE_EXTENSIONS
+from filebrowser.conf import CONCURRENT_MAX_CONNECTIONS, FILE_UPLOAD_CHUNK_SIZE, RESTRICT_FILE_EXTENSIONS, SHOW_DOWNLOAD_BUTTON
 from filebrowser.tasks import check_disk_usage_and_clean_task, document_cleanup_task
+from filebrowser.views import MAX_FILEEDITOR_SIZE
 from hadoop.cluster import is_yarn
 from metadata.catalog_api import (
   _highlight,
@@ -124,10 +125,14 @@ def get_config(request):
   config = get_cluster_config(request.user)
   config['hue_config']['is_admin'] = is_admin(request.user)
   config['hue_config']['is_yarn_enabled'] = is_yarn()
-  config['hue_config']['enable_new_storage_browser'] = ENABLE_NEW_STORAGE_BROWSER.get()
-  config['hue_config']['enable_chunked_file_uploader'] = ENABLE_CHUNKED_FILE_UPLOADER.get()
   config['hue_config']['enable_task_server'] = TASK_SERVER_V2.ENABLED.get()
-  config['hue_config']['restrict_file_extensions'] = RESTRICT_FILE_EXTENSIONS.get()
+  config['storage_browser']['enable_chunked_file_upload'] = ENABLE_CHUNKED_FILE_UPLOADER.get()
+  config['storage_browser']['enable_new_storage_browser'] = ENABLE_NEW_STORAGE_BROWSER.get()
+  config['storage_browser']['restrict_file_extensions'] = RESTRICT_FILE_EXTENSIONS.get()
+  config['storage_browser']['concurrent_max_connection'] = CONCURRENT_MAX_CONNECTIONS.get()
+  config['storage_browser']['file_upload_chunk_size'] = FILE_UPLOAD_CHUNK_SIZE.get()
+  config['storage_browser']['enable_file_download_button'] = SHOW_DOWNLOAD_BUTTON.get()
+  config['storage_browser']['max_file_editor_size'] = MAX_FILEEDITOR_SIZE
   config['clusters'] = list(get_clusters(request.user).values())
   config['documents'] = {
     'types': list(Document2.objects.documents(user=request.user).order_by().values_list('type', flat=True).distinct())

+ 11 - 15
desktop/core/src/desktop/js/apps/storageBrowser/StorageFilePage/StorageFilePage.test.tsx

@@ -22,7 +22,6 @@ import '@testing-library/jest-dom';
 import userEvent from '@testing-library/user-event';
 import { DOWNLOAD_API_URL } from '../../../reactComponents/FileChooser/api';
 import huePubSub from '../../../utils/huePubSub';
-import { hueWindow } from '../../../types/types';
 
 jest.mock('../../../utils/dateTimeUtils', () => ({
   ...jest.requireActual('../../../utils/dateTimeUtils'),
@@ -50,6 +49,16 @@ jest.mock('../../../utils/hooks/useLoadData', () => {
   }));
 });
 
+const mockConfig = {
+  storage_browser: {
+    enable_file_download_button: true,
+    max_file_editor_size: 1000000000
+  }
+};
+jest.mock('../../../config/hueConfig', () => ({
+  getLastKnownConfig: () => mockConfig
+}));
+
 const mockFileStats: FileStats = {
   path: '/path/to/file.txt',
   size: 123456,
@@ -67,19 +76,6 @@ const mockFileName = 'file.txt';
 const mockReload = jest.fn();
 
 describe('StorageFilePage', () => {
-  let oldShowDownloadButton: boolean;
-  let oldMaxFileEditorSize: number;
-  beforeEach(() => {
-    oldShowDownloadButton = (window as hueWindow).SHOW_DOWNLOAD_BUTTON as boolean;
-    oldMaxFileEditorSize = (window as hueWindow).MAX_FILEEDITOR_SIZE as number;
-    (window as hueWindow).SHOW_DOWNLOAD_BUTTON = true;
-    (window as hueWindow).MAX_FILEEDITOR_SIZE = 1000000000;
-  });
-  afterAll(() => {
-    (window as hueWindow).SHOW_DOWNLOAD_BUTTON = oldShowDownloadButton;
-    (window as hueWindow).MAX_FILEEDITOR_SIZE = oldMaxFileEditorSize;
-  });
-
   it('should render file metadata and content', () => {
     render(
       <StorageFilePage fileName={mockFileName} fileStats={mockFileStats} onReload={mockReload} />
@@ -224,7 +220,7 @@ describe('StorageFilePage', () => {
   });
 
   it('should not render the download button when show_download_button is false', () => {
-    (window as hueWindow).SHOW_DOWNLOAD_BUTTON = false;
+    mockConfig.storage_browser.enable_file_download_button = false;
 
     render(
       <StorageFilePage fileName={mockFileName} fileStats={mockFileStats} onReload={mockReload} />

+ 6 - 3
desktop/core/src/desktop/js/apps/storageBrowser/StorageFilePage/StorageFilePage.tsx

@@ -15,7 +15,6 @@
 // limitations under the License.
 
 import React, { useMemo, useState } from 'react';
-import { hueWindow } from 'types/types';
 import {
   BrowserViewType,
   FilePreview,
@@ -39,6 +38,7 @@ import {
 } from '../../../utils/constants/storageBrowser';
 import { Spin } from 'antd';
 import useLoadData from '../../../utils/hooks/useLoadData';
+import { getLastKnownConfig } from '../../../config/hueConfig';
 
 interface StorageFilePageProps {
   onReload: () => void;
@@ -47,7 +47,9 @@ interface StorageFilePageProps {
 }
 
 const StorageFilePage = ({ fileName, fileStats, onReload }: StorageFilePageProps): JSX.Element => {
+  const config = getLastKnownConfig();
   const { t } = i18nReact.useTranslation();
+
   const [isEditing, setIsEditing] = useState<boolean>(false);
   const [fileContent, setFileContent] = useState<FilePreview['contents']>();
 
@@ -114,7 +116,8 @@ const StorageFilePage = ({ fileName, fileStats, onReload }: StorageFilePageProps
 
   const isEditingEnabled =
     !isEditing &&
-    (window as hueWindow).MAX_FILEEDITOR_SIZE > fileStats.size &&
+    config?.storage_browser.max_file_editor_size &&
+    config?.storage_browser.max_file_editor_size > fileStats.size &&
     EDITABLE_FILE_FORMATS[fileType] &&
     fileData?.compression?.toLocaleLowerCase() === 'none';
 
@@ -167,7 +170,7 @@ const StorageFilePage = ({ fileName, fileStats, onReload }: StorageFilePageProps
                   </Button>
                 </>
               )}
-              {(window as hueWindow).SHOW_DOWNLOAD_BUTTON && (
+              {config?.storage_browser.enable_file_download_button && (
                 <a href={`${DOWNLOAD_API_URL}${fileStats.path}`}>
                   <PrimaryButton
                     data-testid="preview--download--button"

+ 13 - 0
desktop/core/src/desktop/js/config/types.ts

@@ -60,6 +60,15 @@ export enum AppType {
   sdkapps = 'sdkapps'
 }
 
+interface StorageBrowserConfig {
+  concurrent_max_connection: number;
+  enable_chunked_file_uploader: boolean;
+  enable_file_download_button: boolean;
+  enable_new_storage_browser: boolean;
+  file_upload_chunk_size: number;
+  max_file_editor_size: number;
+}
+
 export interface HueConfig extends GenericApiResponse {
   app_config: {
     [AppType.browser]?: AppConfig<BrowserInterpreter>;
@@ -83,7 +92,11 @@ export interface HueConfig extends GenericApiResponse {
   hue_config: {
     enable_sharing: boolean;
     collect_usage: boolean;
+    enable_task_server: boolean;
+    is_admin: boolean;
+    is_yarn_enabled: boolean;
   };
+  storage_browser: StorageBrowserConfig;
   hue_version?: string;
   img_version?: string;
   vw_name?: string;

+ 9 - 2
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.tsx

@@ -27,7 +27,11 @@ import StatusStoppedIcon from '@cloudera/cuix-core/icons/react/StatusStoppedIcon
 import StatusErrorIcon from '@cloudera/cuix-core/icons/react/StatusErrorIcon';
 import { UploadItem } from '../../utils/hooks/useFileUpload/util';
 import useFileUpload from '../../utils/hooks/useFileUpload/useFileUpload';
-import { FileUploadStatus } from '../../utils/constants/storageBrowser';
+import {
+  DEFAULT_ENABLE_CHUNK_UPLOAD,
+  FileUploadStatus
+} from '../../utils/constants/storageBrowser';
+import { getLastKnownConfig } from '../../config/hueConfig';
 
 interface FileUploadQueueProps {
   filesQueue: UploadItem[];
@@ -47,11 +51,14 @@ const sortOrder = [
 }, {});
 
 const FileUploadQueue: React.FC<FileUploadQueueProps> = ({ filesQueue, onClose, onComplete }) => {
+  const config = getLastKnownConfig();
+  const isChunkUpload =
+    config?.storage_browser.enable_chunked_file_uploader ?? DEFAULT_ENABLE_CHUNK_UPLOAD;
   const { t } = i18nReact.useTranslation();
   const [isExpanded, setIsExpanded] = useState(true);
 
   const { uploadQueue, onCancel } = useFileUpload(filesQueue, {
-    isChunkUpload: true,
+    isChunkUpload,
     onComplete
   });
 

+ 0 - 2
desktop/core/src/desktop/js/types/types.ts

@@ -59,8 +59,6 @@ export interface hueWindow {
   HUE_LANG?: string;
   HUE_VERSION?: string;
   LOGGED_USERNAME?: string;
-  MAX_FILEEDITOR_SIZE?: number;
-  SHOW_DOWNLOAD_BUTTON?: boolean;
   SQL_ANALYZER_MODE?: string;
   USER_IS_ADMIN?: boolean;
   USER_IS_HUE_ADMIN?: boolean;

+ 2 - 2
desktop/core/src/desktop/js/utils/constants/storageBrowser.ts

@@ -16,8 +16,8 @@
 
 export const DEFAULT_PAGE_SIZE = 50;
 export const DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024; // 5 MiB
-export const DEFAULT_CONCURRENT_UPLOAD = 3;
-export const DEFAULT_CONCURRENT_CHUNK_UPLOAD = 3;
+export const DEFAULT_CONCURRENT_MAX_CONNECTIONS = 3;
+export const DEFAULT_ENABLE_CHUNK_UPLOAD = false;
 
 export enum SupportedFileTypes {
   IMAGE = 'image',

+ 12 - 7
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useChunkUpload.ts

@@ -15,11 +15,12 @@
 // limitations under the License.
 
 import { useEffect, useState } from 'react';
+import { getLastKnownConfig } from '../../../config/hueConfig';
 import useSaveData from '../useSaveData';
 import useQueueProcessor from '../useQueueProcessor';
 import {
   DEFAULT_CHUNK_SIZE,
-  DEFAULT_CONCURRENT_CHUNK_UPLOAD,
+  DEFAULT_CONCURRENT_MAX_CONNECTIONS,
   FileUploadStatus
 } from '../../constants/storageBrowser';
 import useLoadData from '../useLoadData';
@@ -42,14 +43,18 @@ interface UseUploadQueueResponse {
 }
 
 interface ChunkUploadOptions {
+  concurrentProcess?: number;
   onStatusUpdate: (item: UploadItem, newStatus: FileUploadStatus) => void;
   onComplete: () => void;
 }
 
 const useChunkUpload = ({
+  concurrentProcess = DEFAULT_CONCURRENT_MAX_CONNECTIONS,
   onStatusUpdate,
   onComplete
 }: ChunkUploadOptions): UseUploadQueueResponse => {
+  const config = getLastKnownConfig();
+  const chunkSize = config?.storage_browser?.file_upload_chunk_size ?? DEFAULT_CHUNK_SIZE;
   const [pendingItems, setPendingItems] = useState<UploadItem[]>([]);
 
   const { save } = useSaveData(undefined, {
@@ -99,7 +104,7 @@ const useChunkUpload = ({
   };
 
   const onChunksUploadComplete = async (item: UploadItem) => {
-    const { url, payload } = getChunksCompletePayload(item);
+    const { url, payload } = getChunksCompletePayload(item, chunkSize);
     return save(payload, {
       url,
       onSuccess: () => addItemToPending(item)
@@ -107,22 +112,22 @@ const useChunkUpload = ({
   };
 
   const uploadChunk = async (chunkItem: UploadChunkItem) => {
-    const { url, payload } = getChunkItemPayload(chunkItem);
+    const { url, payload } = getChunkItemPayload(chunkItem, chunkSize);
     return save(payload, { url });
   };
 
   const { enqueueAsync } = useQueueProcessor<UploadChunkItem>(uploadChunk, {
-    concurrentProcess: DEFAULT_CONCURRENT_CHUNK_UPLOAD
+    concurrentProcess
   });
 
   const uploadItemInChunks = async (item: UploadItem) => {
-    const chunks = createChunks(item, DEFAULT_CHUNK_SIZE);
+    const chunks = createChunks(item, chunkSize);
     await enqueueAsync(chunks);
     return onChunksUploadComplete(item);
   };
 
   const uploadItemInSingleChunk = (item: UploadItem) => {
-    const { url, payload } = getChunkSinglePayload(item);
+    const { url, payload } = getChunkSinglePayload(item, chunkSize);
     return save(payload, {
       url,
       onSuccess: () => addItemToPending(item)
@@ -131,7 +136,7 @@ const useChunkUpload = ({
 
   const uploadItem = async (item: UploadItem) => {
     onStatusUpdate(item, FileUploadStatus.Uploading);
-    const chunks = getTotalChunk(item.file.size, DEFAULT_CHUNK_SIZE);
+    const chunks = getTotalChunk(item.file.size, chunkSize);
     if (chunks === 1) {
       return uploadItemInSingleChunk(item);
     }

+ 11 - 1
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useFileUpload.ts

@@ -18,7 +18,11 @@ import { useCallback, useEffect, useState } from 'react';
 import useRegularUpload from './useRegularUpload';
 import useChunkUpload from './useChunkUpload';
 import { getNewFileItems, UploadItem } from './util';
-import { FileUploadStatus } from '../../constants/storageBrowser';
+import {
+  DEFAULT_CONCURRENT_MAX_CONNECTIONS,
+  FileUploadStatus
+} from '../../constants/storageBrowser';
+import { getLastKnownConfig } from '../../../config/hueConfig';
 
 interface UseUploadQueueResponse {
   uploadQueue: UploadItem[];
@@ -35,6 +39,10 @@ const useFileUpload = (
   filesQueue: UploadItem[],
   { isChunkUpload = false, onComplete }: UploadQueueOptions
 ): UseUploadQueueResponse => {
+  const config = getLastKnownConfig();
+  const concurrentProcess =
+    config?.storage_browser.concurrent_max_connection ?? DEFAULT_CONCURRENT_MAX_CONNECTIONS;
+
   const [uploadQueue, setUploadQueue] = useState<UploadItem[]>([]);
 
   const onStatusUpdate = (item: UploadItem, newStatus: FileUploadStatus) =>
@@ -52,6 +60,7 @@ const useFileUpload = (
     removeFile: removeFromChunkUpload,
     isLoading: isChunkLoading
   } = useChunkUpload({
+    concurrentProcess,
     onStatusUpdate,
     onComplete
   });
@@ -61,6 +70,7 @@ const useFileUpload = (
     removeFile: removeFromRegularUpload,
     isLoading: isNonChunkLoading
   } = useRegularUpload({
+    concurrentProcess,
     onStatusUpdate,
     onComplete
   });

+ 7 - 2
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useRegularUpload.ts

@@ -16,7 +16,10 @@
 
 import useQueueProcessor from '../useQueueProcessor';
 import { UPLOAD_FILE_URL } from '../../../reactComponents/FileChooser/api';
-import { DEFAULT_CONCURRENT_UPLOAD, FileUploadStatus } from '../../constants/storageBrowser';
+import {
+  DEFAULT_CONCURRENT_MAX_CONNECTIONS,
+  FileUploadStatus
+} from '../../constants/storageBrowser';
 import useSaveData from '../useSaveData';
 import { UploadItem } from './util';
 
@@ -27,11 +30,13 @@ interface UseUploadQueueResponse {
 }
 
 interface UploadQueueOptions {
+  concurrentProcess?: number;
   onStatusUpdate: (item: UploadItem, newStatus: FileUploadStatus) => void;
   onComplete: () => void;
 }
 
 const useRegularUpload = ({
+  concurrentProcess = DEFAULT_CONCURRENT_MAX_CONNECTIONS,
   onStatusUpdate,
   onComplete
 }: UploadQueueOptions): UseUploadQueueResponse => {
@@ -68,7 +73,7 @@ const useRegularUpload = ({
     dequeue: removeFile,
     isLoading
   } = useQueueProcessor<UploadItem>(processUploadItem, {
-    concurrentProcess: DEFAULT_CONCURRENT_UPLOAD,
+    concurrentProcess,
     onSuccess: onComplete
   });
 

+ 15 - 9
desktop/core/src/desktop/js/utils/hooks/useFileUpload/util.ts

@@ -14,7 +14,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { DEFAULT_CHUNK_SIZE, FileUploadStatus } from 'utils/constants/storageBrowser';
+import { FileUploadStatus } from 'utils/constants/storageBrowser';
 import { TaskServerResponse, TaskStatus } from '../../../reactComponents/TaskBrowser/TaskBrowser';
 import { CHUNK_UPLOAD_URL, CHUNK_UPLOAD_COMPLETE_URL } from 'reactComponents/FileChooser/api';
 
@@ -87,11 +87,14 @@ export const getStatusHashMap = (
     {}
   );
 
-export const getChunkItemPayload = (chunkItem: UploadChunkItem): ChunkPayload => {
-  const chunkStart = (chunkItem.chunkIndex ?? 0) * DEFAULT_CHUNK_SIZE;
-  const chunkEnd = Math.min(chunkStart + DEFAULT_CHUNK_SIZE, chunkItem!.file.size);
+export const getChunkItemPayload = (
+  chunkItem: UploadChunkItem,
+  chunkSize: number
+): ChunkPayload => {
+  const chunkStart = (chunkItem.chunkIndex ?? 0) * chunkSize;
+  const chunkEnd = Math.min(chunkStart + chunkSize, chunkItem!.file.size);
 
-  const metaData = getMetaData(chunkItem, DEFAULT_CHUNK_SIZE);
+  const metaData = getMetaData(chunkItem, chunkSize);
   const chunkQueryParams = new URLSearchParams({
     ...metaData,
     qqpartindex: String(chunkItem.chunkIndex),
@@ -105,8 +108,11 @@ export const getChunkItemPayload = (chunkItem: UploadChunkItem): ChunkPayload =>
   return { url, payload };
 };
 
-export const getChunksCompletePayload = (processingItem: UploadItem): ChunkPayload => {
-  const fileMetaData = getMetaData(processingItem, DEFAULT_CHUNK_SIZE);
+export const getChunksCompletePayload = (
+  processingItem: UploadItem,
+  chunkSize: number
+): ChunkPayload => {
+  const fileMetaData = getMetaData(processingItem, chunkSize);
   const payload = new FormData();
   Object.entries(fileMetaData).forEach(([key, value]) => {
     payload.append(key, value);
@@ -114,8 +120,8 @@ export const getChunksCompletePayload = (processingItem: UploadItem): ChunkPaylo
   return { url: CHUNK_UPLOAD_COMPLETE_URL, payload };
 };
 
-export const getChunkSinglePayload = (item: UploadItem): ChunkPayload => {
-  const metaData = getMetaData(item, DEFAULT_CHUNK_SIZE);
+export const getChunkSinglePayload = (item: UploadItem, chunkSize: number): ChunkPayload => {
+  const metaData = getMetaData(item, chunkSize);
 
   const singleChunkParams = Object.fromEntries(
     Object.entries(metaData).filter(([key]) => key !== 'qqtotalparts')

+ 1 - 0
desktop/core/src/desktop/models.py

@@ -1808,6 +1808,7 @@ class ClusterConfig(object):
         'enable_sharing': ENABLE_SHARING.get(),
         'collect_usage': COLLECT_USAGE.get()
       },
+      'storage_browser': {},
       'vw_name': hue_host_name,
       'img_version': img_version,
       'hue_version': version_of_hue

+ 1 - 2
desktop/core/src/desktop/templates/global_js_constants.mako

@@ -31,7 +31,7 @@
   from indexer.conf import ENABLE_NEW_INDEXER
   from jobbrowser.conf import ENABLE_HISTORY_V2, ENABLE_QUERY_BROWSER, ENABLE_HIVE_QUERY_BROWSER, MAX_JOB_FETCH, \
       QUERY_STORE
-  from filebrowser.conf import SHOW_UPLOAD_BUTTON, REMOTE_STORAGE_HOME, MAX_FILE_SIZE_UPLOAD_LIMIT, SHOW_DOWNLOAD_BUTTON
+  from filebrowser.conf import SHOW_UPLOAD_BUTTON, REMOTE_STORAGE_HOME, MAX_FILE_SIZE_UPLOAD_LIMIT
   from filebrowser.views import MAX_FILEEDITOR_SIZE
   from indexer.conf import ENABLE_NEW_INDEXER
   from libsaml.conf import get_logout_redirect_url, CDP_LOGOUT_URL
@@ -211,7 +211,6 @@
 
   window.SHOW_NOTEBOOKS = '${ SHOW_NOTEBOOKS.get() }' === 'True'
   window.SHOW_UPLOAD_BUTTON = '${ hasattr(SHOW_UPLOAD_BUTTON, 'get') and SHOW_UPLOAD_BUTTON.get() }' === 'True'
-  window.SHOW_DOWNLOAD_BUTTON = '${ hasattr(SHOW_DOWNLOAD_BUTTON, 'get') and SHOW_DOWNLOAD_BUTTON.get() }' === 'True'
   window.MAX_FILEEDITOR_SIZE = ${ MAX_FILEEDITOR_SIZE };
 
   window.UPLOAD_CHUNK_SIZE = ${ UPLOAD_CHUNK_SIZE.get() };