Browse Source

[ui-storagebrowser] adds hooks for file upload (#3910)

[ui-storagebrowser] adds hooks for file upload
Ram Prasad Agarwal 11 months ago
parent
commit
130bb0f306

+ 4 - 1
desktop/core/src/desktop/js/reactComponents/FileChooser/api.ts

@@ -13,12 +13,15 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 export const CONTENT_SUMMARY_API_URL = '/api/v1/storage/content_summary';
 export const DOWNLOAD_API_URL = '/filebrowser/download=';
 export const FILESYSTEMS_API_URL = '/api/v1/storage/filesystems';
 export const SAVE_FILE_API_URL = '/filebrowser/save';
 export const VIEWFILES_API_URl = '/api/v1/storage/view=';
-
+export const UPLOAD_FILE_URL = '/filebrowser/upload/file';
+export const CHUNK_UPLOAD_URL = '/filebrowser/upload/chunks/file';
+export const CHUNK_UPLOAD_COMPLETE_URL = '/filebrowser/upload/complete';
 export const CREATE_FILE_API_URL = '/api/v1/storage/create/file/';
 export const CREATE_DIRECTORY_API_URL = '/api/v1/storage/create/directory/';
 export const RENAME_API_URL = '/api/v1/storage/rename/';

+ 80 - 0
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.scss

@@ -0,0 +1,80 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@use 'variables' as vars;
+
+.antd.cuix {
+  .upload-queue {
+    position: fixed;
+    bottom: 0;
+    right: 16px;
+    width: 520px;
+    background-color: vars.$fluidx-white;
+    box-shadow: 0 4px 12px 0 rgba(0, 0, 0, 0.2);
+    z-index: 1000;
+
+    &__header {
+      display: flex;
+      justify-content: space-between;
+      align-items: center;
+      padding: 16px;
+      color: vars.$fluidx-gray-900;
+      background-color: vars.$fluidx-gray-050;
+      cursor: pointer;
+      font-size: 16px;
+    }
+
+    &__list {
+      display: flex;
+      flex-direction: column;
+      max-height: 40vh;
+      overflow: auto;
+      padding: 16px;
+      gap: 16px;
+
+      &__row {
+        display: flex;
+        align-items: center;
+        padding: 8px;
+        font-size: 14px;
+        gap: 8px;
+        border: solid 1px vars.$fluidx-gray-300;
+
+        &__status {
+          display: flex;
+          width: 20px;
+        }
+
+        &__name {
+          display: flex;
+          flex: 1;
+          text-overflow: ellipsis;
+        }
+
+        &__size {
+          display: flex;
+          text-align: right;
+          color: vars.$fluidx-gray-700;
+        }
+
+        &__close {
+          display: flex;
+        }
+      }
+    }
+
+  }
+}

+ 100 - 0
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.test.tsx

@@ -0,0 +1,100 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import React from 'react';
+import { render, screen, fireEvent, waitFor } from '@testing-library/react';
+import '@testing-library/jest-dom';
+import FileUploadQueue from './FileUploadQueue';
+import { FileUploadStatus } from '../../utils/constants/storageBrowser';
+import { UploadItem } from '../../utils/hooks/useFileUpload/util';
+
+const mockFilesQueue: UploadItem[] = [
+  {
+    uuid: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx1',
+    filePath: '/path/to/file1.txt',
+    status: FileUploadStatus.Pending,
+    file: new File([], 'file1.txt')
+  },
+  {
+    uuid: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx2',
+    filePath: '/path/to/file2.txt',
+    status: FileUploadStatus.Pending,
+    file: new File([], 'file2.txt')
+  }
+];
+
+const mockOnCancel = jest.fn();
+jest.mock('../../utils/hooks/useFileUpload/useFileUpload', () => ({
+  __esModule: true,
+  default: jest.fn(() => ({
+    uploadQueue: mockFilesQueue,
+    onCancel: mockOnCancel
+  }))
+}));
+
+describe('FileUploadQueue', () => {
+  it('should render the component with initial files in the queue', () => {
+    const { getByText } = render(
+      <FileUploadQueue filesQueue={mockFilesQueue} onClose={() => {}} onComplete={() => {}} />
+    );
+
+    expect(getByText('file1.txt')).toBeInTheDocument();
+    expect(getByText('file2.txt')).toBeInTheDocument();
+  });
+
+  it('should call onCancel when cancel button is clicked', async () => {
+    const { getAllByTestId } = render(
+      <FileUploadQueue filesQueue={mockFilesQueue} onClose={() => {}} onComplete={() => {}} />
+    );
+
+    const cancelButton = getAllByTestId('upload-queue__list__row__close-icon')[0];
+    fireEvent.click(cancelButton);
+
+    await waitFor(() => {
+      expect(mockOnCancel).toHaveBeenCalled();
+    });
+  });
+
+  it('should toggle the visibility of the queue when the header is clicked', () => {
+    const { getByTestId } = render(
+      <FileUploadQueue filesQueue={mockFilesQueue} onClose={() => {}} onComplete={() => {}} />
+    );
+
+    const header = getByTestId('upload-queue__header');
+    expect(screen.getByText('file1.txt')).toBeInTheDocument();
+    expect(screen.getByText('file2.txt')).toBeInTheDocument();
+
+    fireEvent.click(header!);
+    expect(screen.queryByText('file1.txt')).not.toBeInTheDocument();
+    expect(screen.queryByText('file2.txt')).not.toBeInTheDocument();
+
+    fireEvent.click(header!);
+    expect(screen.getByText('file1.txt')).toBeInTheDocument();
+    expect(screen.getByText('file2.txt')).toBeInTheDocument();
+  });
+
+  it('should render cancel button for files in Pending state', () => {
+    const { getAllByTestId } = render(
+      <FileUploadQueue filesQueue={mockFilesQueue} onClose={() => {}} onComplete={() => {}} />
+    );
+
+    const cancelButtons = getAllByTestId('upload-queue__list__row__close-icon');
+    expect(cancelButtons).toHaveLength(mockFilesQueue.length);
+
+    expect(cancelButtons[0]).toBeInTheDocument();
+    expect(cancelButtons[1]).toBeInTheDocument();
+  });
+});

+ 126 - 0
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.tsx

@@ -0,0 +1,126 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import React, { useState } from 'react';
+import './FileUploadQueue.scss';
+import { Tooltip } from 'antd';
+import CloseIcon from '../../components/icons/CloseIcon';
+import { i18nReact } from '../../utils/i18nReact';
+import formatBytes from '../../utils/formatBytes';
+import StatusPendingIcon from '@cloudera/cuix-core/icons/react/StatusPendingIcon';
+import StatusInProgressIcon from '@cloudera/cuix-core/icons/react/StatusInProgressIcon';
+import StatusSuccessIcon from '@cloudera/cuix-core/icons/react/StatusSuccessIcon';
+import StatusStoppedIcon from '@cloudera/cuix-core/icons/react/StatusStoppedIcon';
+import StatusErrorIcon from '@cloudera/cuix-core/icons/react/StatusErrorIcon';
+import { UploadItem } from '../../utils/hooks/useFileUpload/util';
+import useFileUpload from '../../utils/hooks/useFileUpload/useFileUpload';
+import { FileUploadStatus } from '../../utils/constants/storageBrowser';
+
+interface FileUploadQueueProps {
+  filesQueue: UploadItem[];
+  onClose: () => void;
+  onComplete: () => void;
+}
+
+const sortOrder = [
+  FileUploadStatus.Uploading,
+  FileUploadStatus.Failed,
+  FileUploadStatus.Pending,
+  FileUploadStatus.Canceled,
+  FileUploadStatus.Uploaded
+].reduce((acc: Record<string, number>, status: FileUploadStatus, index: number) => {
+  acc[status] = index + 1;
+  return acc;
+}, {});
+
+const FileUploadQueue: React.FC<FileUploadQueueProps> = ({ filesQueue, onClose, onComplete }) => {
+  const { t } = i18nReact.useTranslation();
+  const [isExpanded, setIsExpanded] = useState(true);
+
+  const { uploadQueue, onCancel } = useFileUpload(filesQueue, {
+    isChunkUpload: true,
+    onComplete
+  });
+
+  const uploadedCount = uploadQueue.filter(
+    item => item.status === FileUploadStatus.Uploaded
+  ).length;
+  const pendingCount = uploadQueue.filter(
+    item => item.status === FileUploadStatus.Pending || item.status === FileUploadStatus.Uploading
+  ).length;
+
+  const statusIcon = {
+    [FileUploadStatus.Pending]: <StatusPendingIcon />,
+    [FileUploadStatus.Uploading]: <StatusInProgressIcon />,
+    [FileUploadStatus.Uploaded]: <StatusSuccessIcon />,
+    [FileUploadStatus.Canceled]: <StatusStoppedIcon />,
+    [FileUploadStatus.Failed]: <StatusErrorIcon />
+  };
+
+  return (
+    <div className="upload-queue cuix antd">
+      <div
+        className="upload-queue__header"
+        data-testid="upload-queue__header"
+        onClick={() => setIsExpanded(prev => !prev)}
+      >
+        {pendingCount > 0
+          ? t('{{count}} file(s) remaining', {
+              count: pendingCount
+            })
+          : t('{{count}} file(s) uploaded', {
+              count: uploadedCount
+            })}
+        <CloseIcon onClick={onClose} height={16} width={16} />
+      </div>
+      {isExpanded && (
+        <div className="upload-queue__list">
+          {uploadQueue
+            .sort((a, b) => sortOrder[a.status] - sortOrder[b.status])
+            .map((row: UploadItem) => (
+              <div key={`${row.filePath}__${row.file.name}`} className="upload-queue__list__row">
+                <Tooltip
+                  title={row.status}
+                  mouseEnterDelay={1.5}
+                  className="upload-queue__list__row__status"
+                >
+                  {statusIcon[row.status]}
+                </Tooltip>
+                <div className="upload-queue__list__row__name">{row.file.name}</div>
+                <div className="upload-queue__list__row__size">{formatBytes(row.file.size)}</div>
+                {row.status === FileUploadStatus.Pending && (
+                  <Tooltip
+                    title={t('Cancel')}
+                    mouseEnterDelay={1.5}
+                    className="upload-queue__list__row__close"
+                  >
+                    <CloseIcon
+                      data-testid="upload-queue__list__row__close-icon"
+                      onClick={() => onCancel(row)}
+                      height={16}
+                      width={16}
+                    />
+                  </Tooltip>
+                )}
+              </div>
+            ))}
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default FileUploadQueue;

+ 16 - 16
desktop/core/src/desktop/js/reactComponents/TaskBrowser/TaskBrowser.tsx

@@ -196,11 +196,26 @@ interface TaskBrowserTableProps {
   handleSchedulePopup: () => void;
 }
 
+export enum TaskStatus {
+  Success = 'SUCCESS',
+  Failure = 'FAILURE',
+  Running = 'RUNNING'
+}
+
+export interface TaskServerResponse {
+  result?: {
+    task_name: string;
+    username: string;
+  };
+  task_id: string;
+  status: TaskStatus;
+}
+
 export const TaskBrowserTable: React.FC<TaskBrowserTableProps> = ({
   ShowTaskLogsHandler,
   handleSchedulePopup
 }) => {
-  const [tasks, setTasks] = useState<Task_Status[]>([]);
+  const [tasks, setTasks] = useState<TaskServerResponse[]>([]);
   const [searchTerm, setSearchTerm] = useState(''); // state for the search term
   const [statusFilter, setStatusFilter] = useState({
     success: false,
@@ -379,21 +394,6 @@ export const TaskBrowserTable: React.FC<TaskBrowserTableProps> = ({
     });
   };
 
-  enum TaskStatus {
-    Success = 'SUCCESS',
-    Failure = 'FAILURE',
-    Running = 'RUNNING'
-  }
-
-  interface Task_Status {
-    result?: {
-      task_name: string;
-      username: string;
-    };
-    task_id: string;
-    status: TaskStatus;
-  }
-
   const filteredTasks = tasks.filter(task => {
     const taskNameMatch = task.result?.task_name?.toLowerCase().includes(searchTerm);
     const userIdMatch = task.result?.username?.toLowerCase().includes(searchTerm);

+ 11 - 0
desktop/core/src/desktop/js/utils/constants/storageBrowser.ts

@@ -15,6 +15,9 @@
 // limitations under the License.
 
 export const DEFAULT_PAGE_SIZE = 50;
+export const DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024; // 5 MiB
+export const DEFAULT_CONCURRENT_UPLOAD = 3;
+export const DEFAULT_CONCURRENT_CHUNK_UPLOAD = 3;
 
 export enum SupportedFileTypes {
   IMAGE = 'image',
@@ -25,6 +28,14 @@ export enum SupportedFileTypes {
   OTHER = 'other'
 }
 
+export enum FileUploadStatus {
+  Pending = 'Pending',
+  Uploading = 'Uploading',
+  Uploaded = 'Uploaded',
+  Canceled = 'Canceled',
+  Failed = 'Failed'
+}
+
 export const SUPPORTED_FILE_EXTENSIONS = {
   png: SupportedFileTypes.IMAGE,
   jpg: SupportedFileTypes.IMAGE,

+ 152 - 0
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useChunkUpload.ts

@@ -0,0 +1,152 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { useEffect, useState } from 'react';
+import useSaveData from '../useSaveData';
+import useQueueProcessor from '../useQueueProcessor';
+import {
+  DEFAULT_CHUNK_SIZE,
+  DEFAULT_CONCURRENT_CHUNK_UPLOAD,
+  FileUploadStatus
+} from '../../constants/storageBrowser';
+import useLoadData from '../useLoadData';
+import { TaskServerResponse, TaskStatus } from '../../../reactComponents/TaskBrowser/TaskBrowser';
+import {
+  createChunks,
+  getChunksCompletePayload,
+  getChunkItemPayload,
+  getChunkSinglePayload,
+  getStatusHashMap,
+  getTotalChunk,
+  UploadChunkItem,
+  UploadItem
+} from './util';
+
+interface UseUploadQueueResponse {
+  addFiles: (item: UploadItem[]) => void;
+  removeFile: (item: UploadItem) => void;
+  isLoading: boolean;
+}
+
+interface ChunkUploadOptions {
+  onStatusUpdate: (item: UploadItem, newStatus: FileUploadStatus) => void;
+  onComplete: () => void;
+}
+
+const useChunkUpload = ({
+  onStatusUpdate,
+  onComplete
+}: ChunkUploadOptions): UseUploadQueueResponse => {
+  const [pendingItems, setPendingItems] = useState<UploadItem[]>([]);
+
+  const { save } = useSaveData(undefined, {
+    postOptions: {
+      qsEncodeData: false,
+      headers: {
+        'Content-Type': 'multipart/form-data'
+      }
+    }
+  });
+
+  const updateItemStatus = (serverResponse: TaskServerResponse[]) => {
+    const statusMap = getStatusHashMap(serverResponse);
+
+    const remainingItems = pendingItems.filter(item => {
+      const status = statusMap[item.uuid];
+      if (status === TaskStatus.Success || status === TaskStatus.Failure) {
+        const ItemStatus =
+          status === TaskStatus.Success ? FileUploadStatus.Uploaded : FileUploadStatus.Failed;
+        onStatusUpdate(item, ItemStatus);
+        return false;
+      }
+      return true;
+    });
+    if (remainingItems.length === 0) {
+      onComplete();
+    }
+    setPendingItems(remainingItems);
+  };
+
+  const { data: tasksStatus } = useLoadData<TaskServerResponse[]>(
+    '/desktop/api2/taskserver/get_taskserver_tasks/',
+    {
+      pollInterval: pendingItems.length ? 5000 : undefined,
+      skip: !pendingItems.length
+    }
+  );
+
+  useEffect(() => {
+    if (tasksStatus) {
+      updateItemStatus(tasksStatus);
+    }
+  }, [tasksStatus]);
+
+  const addItemToPending = (item: UploadItem) => {
+    setPendingItems(prev => [...prev, item]);
+  };
+
+  const onChunksUploadComplete = async (item: UploadItem) => {
+    const { url, payload } = getChunksCompletePayload(item);
+    return save(payload, {
+      url,
+      onSuccess: () => addItemToPending(item)
+    });
+  };
+
+  const uploadChunk = async (chunkItem: UploadChunkItem) => {
+    const { url, payload } = getChunkItemPayload(chunkItem);
+    return save(payload, { url });
+  };
+
+  const { enqueueAsync } = useQueueProcessor<UploadChunkItem>(uploadChunk, {
+    concurrentProcess: DEFAULT_CONCURRENT_CHUNK_UPLOAD
+  });
+
+  const uploadItemInChunks = async (item: UploadItem) => {
+    const chunks = createChunks(item, DEFAULT_CHUNK_SIZE);
+    await enqueueAsync(chunks);
+    return onChunksUploadComplete(item);
+  };
+
+  const uploadItemInSingleChunk = (item: UploadItem) => {
+    const { url, payload } = getChunkSinglePayload(item);
+    return save(payload, {
+      url,
+      onSuccess: () => addItemToPending(item)
+    });
+  };
+
+  const uploadItem = async (item: UploadItem) => {
+    onStatusUpdate(item, FileUploadStatus.Uploading);
+    const chunks = getTotalChunk(item.file.size, DEFAULT_CHUNK_SIZE);
+    if (chunks === 1) {
+      return uploadItemInSingleChunk(item);
+    }
+    return uploadItemInChunks(item);
+  };
+
+  const {
+    enqueue: addFiles,
+    dequeue: removeFile,
+    isLoading
+  } = useQueueProcessor<UploadItem>(uploadItem, {
+    concurrentProcess: 1 // This value must be 1 always
+  });
+
+  return { addFiles, removeFile, isLoading };
+};
+
+export default useChunkUpload;

+ 101 - 0
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useFileUpload.ts

@@ -0,0 +1,101 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { useCallback, useEffect, useState } from 'react';
+import useRegularUpload from './useRegularUpload';
+import useChunkUpload from './useChunkUpload';
+import { getNewFileItems, UploadItem } from './util';
+import { FileUploadStatus } from '../../constants/storageBrowser';
+
+interface UseUploadQueueResponse {
+  uploadQueue: UploadItem[];
+  onCancel: (item: UploadItem) => void;
+  isLoading: boolean;
+}
+
+interface UploadQueueOptions {
+  isChunkUpload?: boolean;
+  onComplete: () => UploadItem[] | void;
+}
+
+const useFileUpload = (
+  filesQueue: UploadItem[],
+  { isChunkUpload = false, onComplete }: UploadQueueOptions
+): UseUploadQueueResponse => {
+  const [uploadQueue, setUploadQueue] = useState<UploadItem[]>([]);
+
+  const onStatusUpdate = (item: UploadItem, newStatus: FileUploadStatus) =>
+    setUploadQueue(prev =>
+      prev.map(queueItem =>
+        queueItem.uuid === item.uuid ? { ...queueItem, status: newStatus } : queueItem
+      )
+    );
+
+  const findQueueItem = (item: UploadItem) =>
+    uploadQueue.filter(queueItem => queueItem.uuid === item.uuid)?.[0];
+
+  const {
+    addFiles: addToChunkUpload,
+    removeFile: removeFromChunkUpload,
+    isLoading: isChunkLoading
+  } = useChunkUpload({
+    onStatusUpdate,
+    onComplete
+  });
+
+  const {
+    addFiles: addToRegularUpload,
+    removeFile: removeFromRegularUpload,
+    isLoading: isNonChunkLoading
+  } = useRegularUpload({
+    onStatusUpdate,
+    onComplete
+  });
+
+  const onCancel = useCallback(
+    (item: UploadItem) => {
+      const queueItem = findQueueItem(item);
+      if (queueItem.status === FileUploadStatus.Pending) {
+        onStatusUpdate(item, FileUploadStatus.Canceled);
+
+        if (isChunkUpload) {
+          removeFromChunkUpload(item);
+        } else {
+          removeFromRegularUpload(item);
+        }
+      }
+    },
+    [isChunkUpload, onStatusUpdate, removeFromChunkUpload, removeFromRegularUpload]
+  );
+
+  useEffect(() => {
+    const newQueueItems = getNewFileItems(filesQueue, uploadQueue);
+
+    if (newQueueItems.length > 0) {
+      setUploadQueue(prev => [...prev, ...newQueueItems]);
+
+      if (isChunkUpload) {
+        addToChunkUpload(newQueueItems);
+      } else {
+        addToRegularUpload(newQueueItems);
+      }
+    }
+  }, [filesQueue, uploadQueue, isChunkUpload]);
+
+  return { uploadQueue, onCancel, isLoading: isChunkLoading || isNonChunkLoading };
+};
+
+export default useFileUpload;

+ 78 - 0
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useRegularUpload.ts

@@ -0,0 +1,78 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import useQueueProcessor from '../useQueueProcessor';
+import { UPLOAD_FILE_URL } from '../../../reactComponents/FileChooser/api';
+import { DEFAULT_CONCURRENT_UPLOAD, FileUploadStatus } from '../../constants/storageBrowser';
+import useSaveData from '../useSaveData';
+import { UploadItem } from './util';
+
+interface UseUploadQueueResponse {
+  addFiles: (item: UploadItem[]) => void;
+  removeFile: (item: UploadItem) => void;
+  isLoading: boolean;
+}
+
+interface UploadQueueOptions {
+  onStatusUpdate: (item: UploadItem, newStatus: FileUploadStatus) => void;
+  onComplete: () => void;
+}
+
+const useRegularUpload = ({
+  onStatusUpdate,
+  onComplete
+}: UploadQueueOptions): UseUploadQueueResponse => {
+  const { save } = useSaveData(undefined, {
+    postOptions: {
+      qsEncodeData: false,
+      headers: {
+        'Content-Type': 'multipart/form-data'
+      }
+    }
+  });
+
+  const processUploadItem = async (item: UploadItem) => {
+    onStatusUpdate(item, FileUploadStatus.Uploading);
+
+    const url = `${UPLOAD_FILE_URL}?dest=${item.filePath}`;
+
+    const payload = new FormData();
+    payload.append('hdfs_file', item.file);
+
+    return save(payload, {
+      url,
+      onSuccess: () => {
+        onStatusUpdate(item, FileUploadStatus.Uploaded);
+      },
+      onError: () => {
+        onStatusUpdate(item, FileUploadStatus.Failed);
+      }
+    });
+  };
+
+  const {
+    enqueue: addFiles,
+    dequeue: removeFile,
+    isLoading
+  } = useQueueProcessor<UploadItem>(processUploadItem, {
+    concurrentProcess: DEFAULT_CONCURRENT_UPLOAD,
+    onSuccess: onComplete
+  });
+
+  return { addFiles, removeFile, isLoading };
+};
+
+export default useRegularUpload;

+ 131 - 0
desktop/core/src/desktop/js/utils/hooks/useFileUpload/util.ts

@@ -0,0 +1,131 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { DEFAULT_CHUNK_SIZE, FileUploadStatus } from 'utils/constants/storageBrowser';
+import { TaskServerResponse, TaskStatus } from '../../../reactComponents/TaskBrowser/TaskBrowser';
+import { CHUNK_UPLOAD_URL, CHUNK_UPLOAD_COMPLETE_URL } from 'reactComponents/FileChooser/api';
+
+export interface UploadItem {
+  uuid: string;
+  filePath: string;
+  file: File;
+  status: FileUploadStatus;
+}
+
+export interface UploadMetaData {
+  qqtotalparts: string;
+  qqtotalfilesize: string;
+  qqfilename: string;
+  inputName: string;
+  dest: string;
+  qquuid: string;
+}
+
+export interface UploadChunkItem extends UploadItem {
+  chunkIndex: number;
+}
+
+interface ChunkPayload {
+  url: string;
+  payload: FormData;
+}
+
+export const getNewFileItems = (newQueue: UploadItem[], oldQueue: UploadItem[]): UploadItem[] => {
+  return newQueue.filter(
+    newItem =>
+      !oldQueue.some(
+        oldItem => oldItem.file.name === newItem.file.name && oldItem.filePath === newItem.filePath
+      )
+  );
+};
+
+export const getTotalChunk = (fileSize: number, DEFAULT_CHUNK_SIZE: number): number => {
+  return Math.ceil(fileSize / DEFAULT_CHUNK_SIZE);
+};
+
+export const getMetaData = (item: UploadItem, DEFAULT_CHUNK_SIZE: number): UploadMetaData => ({
+  qqtotalparts: String(getTotalChunk(item.file.size, DEFAULT_CHUNK_SIZE)),
+  qqtotalfilesize: String(item.file.size),
+  qqfilename: item.file.name,
+  inputName: 'hdfs_file',
+  dest: item.filePath,
+  qquuid: item.uuid
+});
+
+export const createChunks = (item: UploadItem, chunkSize: number): UploadChunkItem[] => {
+  const totalChunks = getTotalChunk(item.file.size, chunkSize);
+
+  const chunks = Array.from({ length: totalChunks }, (_, i) => ({
+    ...item,
+    chunkIndex: i
+  }));
+
+  return chunks;
+};
+
+export const getStatusHashMap = (
+  serverResponse: TaskServerResponse[]
+): Record<string, TaskStatus> =>
+  serverResponse.reduce(
+    (acc, row: TaskServerResponse) => ({
+      ...acc,
+      [row.task_id]: row.status
+    }),
+    {}
+  );
+
+export const getChunkItemPayload = (chunkItem: UploadChunkItem): ChunkPayload => {
+  const chunkStart = (chunkItem.chunkIndex ?? 0) * DEFAULT_CHUNK_SIZE;
+  const chunkEnd = Math.min(chunkStart + DEFAULT_CHUNK_SIZE, chunkItem!.file.size);
+
+  const metaData = getMetaData(chunkItem, DEFAULT_CHUNK_SIZE);
+  const chunkQueryParams = new URLSearchParams({
+    ...metaData,
+    qqpartindex: String(chunkItem.chunkIndex),
+    qqpartbyteoffset: String(chunkStart),
+    qqchunksize: String(chunkEnd - chunkStart)
+  }).toString();
+  const url = `${CHUNK_UPLOAD_URL}?${chunkQueryParams}`;
+
+  const payload = new FormData();
+  payload.append('hdfs_file', chunkItem!.file.slice(chunkStart, chunkEnd));
+  return { url, payload };
+};
+
+export const getChunksCompletePayload = (processingItem: UploadItem): ChunkPayload => {
+  const fileMetaData = getMetaData(processingItem, DEFAULT_CHUNK_SIZE);
+  const payload = new FormData();
+  Object.entries(fileMetaData).forEach(([key, value]) => {
+    payload.append(key, value);
+  });
+  return { url: CHUNK_UPLOAD_COMPLETE_URL, payload };
+};
+
+export const getChunkSinglePayload = (item: UploadItem): ChunkPayload => {
+  const metaData = getMetaData(item, DEFAULT_CHUNK_SIZE);
+
+  const singleChunkParams = Object.fromEntries(
+    Object.entries(metaData).filter(([key]) => key !== 'qqtotalparts')
+  );
+
+  const queryParams = new URLSearchParams(singleChunkParams).toString();
+  const url = `${CHUNK_UPLOAD_URL}?${queryParams}`;
+
+  const payload = new FormData();
+  payload.append('hdfs_file', item.file);
+
+  return { url, payload };
+};

+ 18 - 0
desktop/core/src/desktop/js/utils/hooks/useLoadData.ts

@@ -23,6 +23,7 @@ export interface Options<T, U> {
   skip?: boolean;
   onSuccess?: (data: T) => void;
   onError?: (error: Error) => void;
+  pollInterval?: number;
 }
 
 interface UseLoadDataProps<T> {
@@ -79,6 +80,23 @@ const useLoadData = <T, U = unknown>(
     [url, localOptions, fetchOptions]
   );
 
+  useEffect(() => {
+    let interval: NodeJS.Timeout | undefined;
+
+    if (localOptions?.pollInterval) {
+      interval = setInterval(() => {
+        loadData();
+      }, localOptions.pollInterval);
+    }
+
+    // Cleanup interval if pollInterval is undefined or when the component unmounts
+    return () => {
+      if (interval) {
+        clearInterval(interval);
+      }
+    };
+  }, [localOptions?.pollInterval, loadData]);
+
   useEffect(() => {
     loadData();
   }, [loadData]);

+ 14 - 0
desktop/core/src/desktop/js/utils/hooks/useQueueProcessor.ts

@@ -19,6 +19,7 @@ import { useState, useEffect } from 'react';
 interface UseQueueProcessorResult<T> {
   queue: T[];
   enqueue: (newItems: T[]) => void;
+  enqueueAsync: (newItems: T[]) => Promise<void>;
   dequeue: (item: T) => void;
   isLoading: boolean;
 }
@@ -40,6 +41,18 @@ const useQueueProcessor = <T>(
     setQueue(prevQueue => [...prevQueue, ...newItems]);
   };
 
+  const enqueueAsync = (newItems: T[]) => {
+    enqueue(newItems);
+    return new Promise<void>(resolve => {
+      const interval = setInterval(() => {
+        if (queue.length === 0 && processingQueue.length === 0) {
+          clearInterval(interval);
+          resolve();
+        }
+      }, 100);
+    });
+  };
+
   const dequeue = (item: T) => {
     setQueue(prev => prev.filter(i => i !== item));
   };
@@ -72,6 +85,7 @@ const useQueueProcessor = <T>(
     queue,
     isLoading,
     enqueue,
+    enqueueAsync,
     dequeue
   };
 };

+ 6 - 4
desktop/core/src/desktop/js/utils/hooks/useSaveData.ts

@@ -18,6 +18,7 @@ import { useCallback, useEffect, useMemo, useState } from 'react';
 import { ApiFetchOptions, post } from '../../api/utils';
 
 interface saveOptions<T> {
+  url?: string;
   onSuccess?: (data: T) => void;
   onError?: (error: Error) => void;
 }
@@ -31,7 +32,7 @@ interface UseSaveData<T, U> {
   data?: T;
   loading: boolean;
   error?: Error;
-  save: (body: U, saveOption: saveOptions<T>) => void;
+  save: (body: U, saveOption?: saveOptions<T>) => void;
 }
 
 const useSaveData = <T, U = unknown>(url?: string, options?: Options<T>): UseSaveData<T, U> => {
@@ -51,17 +52,18 @@ const useSaveData = <T, U = unknown>(url?: string, options?: Options<T>): UseSav
   );
 
   const saveData = useCallback(
-    async (body: U, saveOptions: saveOptions<T>) => {
+    async (body: U, saveOptions?: saveOptions<T>) => {
       // Avoid Posting data if the skip option is true
       // or if the URL is not provided
-      if (options?.skip || !url) {
+      const apiUrl = saveOptions?.url ?? url;
+      if (options?.skip || !apiUrl) {
         return;
       }
       setLoading(true);
       setError(undefined);
 
       try {
-        const response = await post<T, U>(url, body, postOptions);
+        const response = await post<T, U>(apiUrl, body, postOptions);
         setData(response);
         if (saveOptions?.onSuccess) {
           saveOptions.onSuccess(response);