Преглед на файлове

[ui-storagebrowser] add progressBar in file upload (#4059)

Ram Prasad Agarwal преди 8 месеца
родител
ревизия
382177d86a
променени са 17 файла, в които са добавени 679 реда и са изтрити 467 реда
  1. 4 8
      desktop/core/src/desktop/js/apps/storageBrowser/StorageDirectoryPage/StorageDirectoryPage.tsx
  2. 1 32
      desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.scss
  3. 5 31
      desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.test.tsx
  4. 20 58
      desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.tsx
  5. 61 0
      desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadRow/FileUploadRow.scss
  6. 66 0
      desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadRow/FileUploadRow.test.tsx
  7. 71 0
      desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadRow/FileUploadRow.tsx
  8. 0 8
      desktop/core/src/desktop/js/utils/constants/storageBrowser.ts
  9. 70 0
      desktop/core/src/desktop/js/utils/hooks/useFileUpload/types.ts
  10. 108 118
      desktop/core/src/desktop/js/utils/hooks/useFileUpload/useChunkUpload.ts
  11. 42 34
      desktop/core/src/desktop/js/utils/hooks/useFileUpload/useFileUpload.ts
  12. 23 17
      desktop/core/src/desktop/js/utils/hooks/useFileUpload/useRegularUpload.ts
  13. 0 137
      desktop/core/src/desktop/js/utils/hooks/useFileUpload/util.ts
  14. 166 0
      desktop/core/src/desktop/js/utils/hooks/useFileUpload/utils.ts
  15. 26 0
      desktop/core/src/desktop/js/utils/hooks/useQueueProcessor/useQueueProcessor.test.tsx
  16. 7 16
      desktop/core/src/desktop/js/utils/hooks/useQueueProcessor/useQueueProcessor.ts
  17. 9 8
      desktop/core/src/desktop/js/utils/hooks/useSaveData/useSaveData.ts

+ 4 - 8
desktop/core/src/desktop/js/apps/storageBrowser/StorageDirectoryPage/StorageDirectoryPage.tsx

@@ -36,14 +36,10 @@ import formatBytes from '../../../utils/formatBytes';
 import './StorageDirectoryPage.scss';
 import { formatTimestamp } from '../../../utils/dateTimeUtils';
 import useLoadData from '../../../utils/hooks/useLoadData/useLoadData';
-import {
-  DEFAULT_PAGE_SIZE,
-  DEFAULT_POLLING_TIME,
-  FileUploadStatus
-} from '../../../utils/constants/storageBrowser';
+import { DEFAULT_PAGE_SIZE, DEFAULT_POLLING_TIME } from '../../../utils/constants/storageBrowser';
 import DragAndDrop from '../../../reactComponents/DragAndDrop/DragAndDrop';
 import UUID from '../../../utils/string/UUID';
-import { UploadItem } from '../../../utils/hooks/useFileUpload/util';
+import { RegularFile, FileStatus } from '../../../utils/hooks/useFileUpload/types';
 import FileUploadQueue from '../../../reactComponents/FileUploadQueue/FileUploadQueue';
 import { useWindowSize } from '../../../utils/hooks/useWindowSize/useWindowSize';
 import LoadingErrorWrapper from '../../../reactComponents/LoadingErrorWrapper/LoadingErrorWrapper';
@@ -74,7 +70,7 @@ const StorageDirectoryPage = ({
 }: StorageDirectoryPageProps): JSX.Element => {
   const [loadingFiles, setLoadingFiles] = useState<boolean>(false);
   const [selectedFiles, setSelectedFiles] = useState<StorageDirectoryTableData[]>([]);
-  const [filesToUpload, setFilesToUpload] = useState<UploadItem[]>([]);
+  const [filesToUpload, setFilesToUpload] = useState<RegularFile[]>([]);
   const [polling, setPolling] = useState<boolean>(false);
 
   const [pageSize, setPageSize] = useState<number>(DEFAULT_PAGE_SIZE);
@@ -152,7 +148,7 @@ const StorageDirectoryPage = ({
         file,
         filePath: fileStats.path,
         uuid: UUID(),
-        status: FileUploadStatus.Pending
+        status: FileStatus.Pending
       };
     });
     setPolling(true);

+ 1 - 32
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.scss

@@ -44,37 +44,6 @@
       overflow: auto;
       padding: 16px;
       gap: 16px;
-
-      &__row {
-        display: flex;
-        align-items: center;
-        padding: 8px;
-        font-size: 14px;
-        gap: 8px;
-        border: solid 1px vars.$fluidx-gray-300;
-
-        &__status {
-          display: flex;
-          width: 20px;
-        }
-
-        &__name {
-          display: flex;
-          flex: 1;
-          text-overflow: ellipsis;
-        }
-
-        &__size {
-          display: flex;
-          text-align: right;
-          color: vars.$fluidx-gray-700;
-        }
-
-        &__close {
-          display: flex;
-        }
-      }
     }
-
   }
-}
+}

+ 5 - 31
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.test.tsx

@@ -15,23 +15,22 @@
 // limitations under the License.
 
 import React from 'react';
-import { render, screen, fireEvent, waitFor } from '@testing-library/react';
+import { render, screen, fireEvent } from '@testing-library/react';
 import '@testing-library/jest-dom';
 import FileUploadQueue from './FileUploadQueue';
-import { FileUploadStatus } from '../../utils/constants/storageBrowser';
-import { UploadItem } from '../../utils/hooks/useFileUpload/util';
+import { FileStatus, RegularFile } from '../../utils/hooks/useFileUpload/types';
 
-const mockFilesQueue: UploadItem[] = [
+const mockFilesQueue: RegularFile[] = [
   {
     uuid: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx1',
     filePath: '/path/to/file1.txt',
-    status: FileUploadStatus.Pending,
+    status: FileStatus.Pending,
     file: new File([], 'file1.txt')
   },
   {
     uuid: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx2',
     filePath: '/path/to/file2.txt',
-    status: FileUploadStatus.Pending,
+    status: FileStatus.Pending,
     file: new File([], 'file2.txt')
   }
 ];
@@ -55,19 +54,6 @@ describe('FileUploadQueue', () => {
     expect(getByText('file2.txt')).toBeInTheDocument();
   });
 
-  it('should call onCancel when cancel button is clicked', async () => {
-    const { getAllByTestId } = render(
-      <FileUploadQueue filesQueue={mockFilesQueue} onClose={() => {}} onComplete={() => {}} />
-    );
-
-    const cancelButton = getAllByTestId('upload-queue__list__row__close-icon')[0];
-    fireEvent.click(cancelButton);
-
-    await waitFor(() => {
-      expect(mockOnCancel).toHaveBeenCalled();
-    });
-  });
-
   it('should toggle the visibility of the queue when the header is clicked', () => {
     const { getByTestId } = render(
       <FileUploadQueue filesQueue={mockFilesQueue} onClose={() => {}} onComplete={() => {}} />
@@ -85,16 +71,4 @@ describe('FileUploadQueue', () => {
     expect(screen.getByText('file1.txt')).toBeInTheDocument();
     expect(screen.getByText('file2.txt')).toBeInTheDocument();
   });
-
-  it('should render cancel button for files in Pending state', () => {
-    const { getAllByTestId } = render(
-      <FileUploadQueue filesQueue={mockFilesQueue} onClose={() => {}} onComplete={() => {}} />
-    );
-
-    const cancelButtons = getAllByTestId('upload-queue__list__row__close-icon');
-    expect(cancelButtons).toHaveLength(mockFilesQueue.length);
-
-    expect(cancelButtons[0]).toBeInTheDocument();
-    expect(cancelButtons[1]).toBeInTheDocument();
-  });
 });

+ 20 - 58
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.tsx

@@ -15,37 +15,29 @@
 // limitations under the License.
 
 import React, { useState } from 'react';
-import './FileUploadQueue.scss';
-import { Tooltip } from 'antd';
 import CloseIcon from '../../components/icons/CloseIcon';
 import { i18nReact } from '../../utils/i18nReact';
-import formatBytes from '../../utils/formatBytes';
-import StatusPendingIcon from '@cloudera/cuix-core/icons/react/StatusPendingIcon';
-import StatusInProgressIcon from '@cloudera/cuix-core/icons/react/StatusInProgressIcon';
-import StatusSuccessIcon from '@cloudera/cuix-core/icons/react/StatusSuccessIcon';
-import StatusStoppedIcon from '@cloudera/cuix-core/icons/react/StatusStoppedIcon';
-import StatusErrorIcon from '@cloudera/cuix-core/icons/react/StatusErrorIcon';
-import { UploadItem } from '../../utils/hooks/useFileUpload/util';
+import { RegularFile, FileStatus } from '../../utils/hooks/useFileUpload/types';
 import useFileUpload from '../../utils/hooks/useFileUpload/useFileUpload';
-import {
-  DEFAULT_ENABLE_CHUNK_UPLOAD,
-  FileUploadStatus
-} from '../../utils/constants/storageBrowser';
+import { DEFAULT_ENABLE_CHUNK_UPLOAD } from '../../utils/constants/storageBrowser';
 import { getLastKnownConfig } from '../../config/hueConfig';
+import FileUploadRow from './FileUploadRow/FileUploadRow';
+
+import './FileUploadQueue.scss';
 
 interface FileUploadQueueProps {
-  filesQueue: UploadItem[];
+  filesQueue: RegularFile[];
   onClose: () => void;
   onComplete: () => void;
 }
 
 const sortOrder = [
-  FileUploadStatus.Uploading,
-  FileUploadStatus.Failed,
-  FileUploadStatus.Pending,
-  FileUploadStatus.Canceled,
-  FileUploadStatus.Uploaded
-].reduce((acc: Record<string, number>, status: FileUploadStatus, index: number) => {
+  FileStatus.Uploading,
+  FileStatus.Failed,
+  FileStatus.Pending,
+  FileStatus.Cancelled,
+  FileStatus.Uploaded
+].reduce((acc: Record<string, number>, status: FileStatus, index: number) => {
   acc[status] = index + 1;
   return acc;
 }, {});
@@ -64,21 +56,11 @@ const FileUploadQueue: React.FC<FileUploadQueueProps> = ({ filesQueue, onClose,
     onComplete
   });
 
-  const uploadedCount = uploadQueue.filter(
-    item => item.status === FileUploadStatus.Uploaded
-  ).length;
+  const uploadedCount = uploadQueue.filter(item => item.status === FileStatus.Uploaded).length;
   const pendingCount = uploadQueue.filter(
-    item => item.status === FileUploadStatus.Pending || item.status === FileUploadStatus.Uploading
+    item => item.status === FileStatus.Pending || item.status === FileStatus.Uploading
   ).length;
 
-  const statusIcon = {
-    [FileUploadStatus.Pending]: <StatusPendingIcon />,
-    [FileUploadStatus.Uploading]: <StatusInProgressIcon />,
-    [FileUploadStatus.Uploaded]: <StatusSuccessIcon />,
-    [FileUploadStatus.Canceled]: <StatusStoppedIcon />,
-    [FileUploadStatus.Failed]: <StatusErrorIcon />
-  };
-
   return (
     <div className="upload-queue cuix antd">
       <div
@@ -99,32 +81,12 @@ const FileUploadQueue: React.FC<FileUploadQueueProps> = ({ filesQueue, onClose,
         <div className="upload-queue__list">
           {uploadQueue
             .sort((a, b) => sortOrder[a.status] - sortOrder[b.status])
-            .map((row: UploadItem) => (
-              <div key={`${row.filePath}__${row.file.name}`} className="upload-queue__list__row">
-                <Tooltip
-                  title={row.status}
-                  mouseEnterDelay={1.5}
-                  className="upload-queue__list__row__status"
-                >
-                  {statusIcon[row.status]}
-                </Tooltip>
-                <div className="upload-queue__list__row__name">{row.file.name}</div>
-                <div className="upload-queue__list__row__size">{formatBytes(row.file.size)}</div>
-                {row.status === FileUploadStatus.Pending && (
-                  <Tooltip
-                    title={t('Cancel')}
-                    mouseEnterDelay={1.5}
-                    className="upload-queue__list__row__close"
-                  >
-                    <CloseIcon
-                      data-testid="upload-queue__list__row__close-icon"
-                      onClick={() => onCancel(row)}
-                      height={16}
-                      width={16}
-                    />
-                  </Tooltip>
-                )}
-              </div>
+            .map((row: RegularFile) => (
+              <FileUploadRow
+                key={`${row.filePath}__${row.file.name}`}
+                data={row}
+                onCancel={() => onCancel(row)}
+              />
             ))}
         </div>
       )}

+ 61 - 0
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadRow/FileUploadRow.scss

@@ -0,0 +1,61 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@use 'variables' as vars;
+
+.antd.cuix {
+  .hue-upload-queue-row {
+    display: flex;
+    flex-direction: column;
+    border: solid 1px vars.$fluidx-gray-300;
+
+    &__container {
+      display: flex;
+      align-items: center;
+      font-size: 14px;
+      padding: 8px;
+      gap: 8px;
+    }
+
+    &__status {
+      display: flex;
+      width: 20px;
+    }
+
+    &__name {
+      display: flex;
+      flex: 1;
+      text-overflow: ellipsis;
+    }
+
+    &__size {
+      display: flex;
+      text-align: right;
+      color: vars.$fluidx-gray-700;
+    }
+
+    &__close-icon {
+      cursor: pointer;
+      height: 16px;
+      width: 16px;
+    }
+
+    &__progressbar {
+      height: 2px;
+      background-color: vars.$fluidx-blue-500;
+    }
+  }
+}

+ 66 - 0
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadRow/FileUploadRow.test.tsx

@@ -0,0 +1,66 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import React from 'react';
+import { render, fireEvent, waitFor } from '@testing-library/react';
+import '@testing-library/jest-dom';
+import { RegularFile, FileStatus } from '../../../utils/hooks/useFileUpload/types';
+import FileUploadRow from './FileUploadRow';
+
+const mockUploadRow: RegularFile = {
+  uuid: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx1',
+  filePath: '/path/to/file1.txt',
+  status: FileStatus.Pending,
+  file: new File(['mock test file'], 'file1.txt'),
+  progress: 0
+};
+const mockOnCancel = jest.fn();
+
+describe('FileUploadRow', () => {
+  it('should render the row with name', () => {
+    const { getByText } = render(<FileUploadRow data={mockUploadRow} onCancel={mockOnCancel} />);
+
+    expect(getByText('file1.txt')).toBeInTheDocument();
+    expect(getByText('14 Bytes')).toBeInTheDocument();
+  });
+
+  it('should call onCancel when cancel button is clicked', async () => {
+    const { getByTestId } = render(<FileUploadRow data={mockUploadRow} onCancel={mockOnCancel} />);
+
+    const cancelButton = getByTestId('hue-upload-queue-row__close-icon');
+    fireEvent.click(cancelButton);
+
+    await waitFor(() => {
+      expect(mockOnCancel).toHaveBeenCalled();
+    });
+  });
+
+  it('should hide cancel button for files is not in Pending state', () => {
+    const mockData = { ...mockUploadRow, status: FileStatus.Failed };
+    const { queryByTestId } = render(<FileUploadRow data={mockData} onCancel={mockOnCancel} />);
+
+    const cancelButtons = queryByTestId('hue-upload-queue-row__close-icon');
+    expect(cancelButtons).not.toBeInTheDocument();
+  });
+
+  it('should show progress bar when file is in uploading state', () => {
+    const mockData = { ...mockUploadRow, status: FileStatus.Uploading, progress: 10 };
+    const { getByRole } = render(<FileUploadRow data={mockData} onCancel={mockOnCancel} />);
+    const progressBar = getByRole('hue-upload-queue-row__progressbar');
+    expect(progressBar).toBeInTheDocument();
+    expect(progressBar).toHaveStyle('width: 10%');
+  });
+});

+ 71 - 0
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadRow/FileUploadRow.tsx

@@ -0,0 +1,71 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import React from 'react';
+import './FileUploadRow.scss';
+import { Tooltip } from 'antd';
+import CloseIcon from '../../../components/icons/CloseIcon';
+import formatBytes from '../../../utils/formatBytes';
+import StatusPendingIcon from '@cloudera/cuix-core/icons/react/StatusPendingIcon';
+import StatusInProgressIcon from '@cloudera/cuix-core/icons/react/StatusInProgressIcon';
+import StatusSuccessIcon from '@cloudera/cuix-core/icons/react/StatusSuccessIcon';
+import StatusStoppedIcon from '@cloudera/cuix-core/icons/react/StatusStoppedIcon';
+import StatusErrorIcon from '@cloudera/cuix-core/icons/react/StatusErrorIcon';
+import { RegularFile, FileStatus } from '../../../utils/hooks/useFileUpload/types';
+import { i18nReact } from '../../../utils/i18nReact';
+
+interface FileUploadRowProps {
+  data: RegularFile;
+  onCancel: () => void;
+}
+
+const statusIcon = {
+  [FileStatus.Pending]: <StatusPendingIcon />,
+  [FileStatus.Uploading]: <StatusInProgressIcon />,
+  [FileStatus.Uploaded]: <StatusSuccessIcon />,
+  [FileStatus.Cancelled]: <StatusStoppedIcon />,
+  [FileStatus.Failed]: <StatusErrorIcon />
+};
+
+const FileUploadRow: React.FC<FileUploadRowProps> = ({ data, onCancel }) => {
+  const { t } = i18nReact.useTranslation();
+
+  return (
+    <div key={`${data.filePath}__${data.file.name}`} className="hue-upload-queue-row">
+      <div className="hue-upload-queue-row__container">
+        <Tooltip title={data.status} mouseEnterDelay={1.5} className="hue-upload-queue-row__status">
+          {statusIcon[data.status]}
+        </Tooltip>
+        <div className="hue-upload-queue-row__name">{data.file.name}</div>
+        <div className="hue-upload-queue-row__size">{formatBytes(data.file.size)}</div>
+        {data.status === FileStatus.Pending && (
+          <Tooltip title={t('Cancel')} mouseEnterDelay={1.5}>
+            <CloseIcon data-testid="hue-upload-queue-row__close-icon" onClick={onCancel} />
+          </Tooltip>
+        )}
+      </div>
+      <div
+        className="hue-upload-queue-row__progressbar"
+        role="hue-upload-queue-row__progressbar"
+        style={{
+          width: `${data.status === FileStatus.Uploading ? data.progress : 0}%`
+        }}
+      />
+    </div>
+  );
+};
+
+export default FileUploadRow;

+ 0 - 8
desktop/core/src/desktop/js/utils/constants/storageBrowser.ts

@@ -31,14 +31,6 @@ export enum SupportedFileTypes {
   OTHER = 'other'
 }
 
-export enum FileUploadStatus {
-  Pending = 'Pending',
-  Uploading = 'Uploading',
-  Uploaded = 'Uploaded',
-  Canceled = 'Canceled',
-  Failed = 'Failed'
-}
-
 export const SUPPORTED_FILE_EXTENSIONS: Record<string, SupportedFileTypes> = {
   png: SupportedFileTypes.IMAGE,
   jpg: SupportedFileTypes.IMAGE,

+ 70 - 0
desktop/core/src/desktop/js/utils/hooks/useFileUpload/types.ts

@@ -0,0 +1,70 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export enum FileStatus {
+  Pending = 'pending',
+  Uploading = 'uploading',
+  Uploaded = 'uploaded',
+  Cancelled = 'cancelled',
+  Failed = 'failed'
+}
+
+// Interface for file upload as a whole file in one single request.
+export interface RegularFile {
+  uuid: string;
+  filePath: string;
+  file: File;
+  status: FileStatus;
+  progress?: number;
+  error?: Error;
+}
+
+// Interface for file upload in chunks.
+// One RegularFile can be broken down into multiple ChunkedFile.
+// And each ChunkedFile can be uploaded independently and combined at backed server
+export interface ChunkedFile extends Omit<RegularFile, 'file'> {
+  file: Blob; // storing only part of the file to avoid big file duplication
+  fileName: string;
+  totalSize: number;
+  chunkNumber: number;
+  chunkStartOffset: number;
+  chunkEndOffset: number;
+  totalChunks: number;
+}
+
+export interface FileVariables extends Partial<Omit<RegularFile, 'uuid' | 'filePath' | 'file'>> {}
+
+export interface FileChunkMetaData {
+  qqtotalparts: string;
+  qqtotalfilesize: string;
+  qqfilename: string;
+  inputName: string;
+  dest: string;
+  qquuid: string;
+}
+
+export interface ChunkedFilesInProgress {
+  [uuid: string]: {
+    chunkNumber: number;
+    progress: number;
+    chunkSize: number;
+  }[];
+}
+
+export interface FileUploadApiPayload {
+  url: string;
+  payload: FormData;
+}

+ 108 - 118
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useChunkUpload.ts

@@ -14,181 +14,171 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { useEffect, useState } from 'react';
+import { useState } from 'react';
 import { getLastKnownConfig } from '../../../config/hueConfig';
 import useSaveData from '../useSaveData/useSaveData';
 import useQueueProcessor from '../useQueueProcessor/useQueueProcessor';
 import {
   DEFAULT_CHUNK_SIZE,
-  DEFAULT_CONCURRENT_MAX_CONNECTIONS,
-  FileUploadStatus
+  DEFAULT_CONCURRENT_MAX_CONNECTIONS
 } from '../../constants/storageBrowser';
 import useLoadData from '../useLoadData/useLoadData';
 import { TaskServerResponse, TaskStatus } from '../../../reactComponents/TaskBrowser/TaskBrowser';
 import {
-  createChunks,
   getChunksCompletePayload,
+  getItemProgress,
+  getItemsTotalProgress,
   getChunkItemPayload,
-  getChunkSinglePayload,
+  createChunks,
   getStatusHashMap,
-  getTotalChunk,
-  UploadChunkItem,
-  UploadItem
-} from './util';
-import { get } from '../../../api/utils';
-import { UPLOAD_AVAILABLE_SPACE_URL } from '../../../apps/storageBrowser/api';
-
-interface UseUploadQueueResponse {
-  addFiles: (item: UploadItem[]) => void;
-  removeFile: (item: UploadItem) => void;
+  addChunkToInProcess,
+  isSpaceAvailableInServer,
+  isAllChunksOfFileUploaded
+} from './utils';
+import {
+  RegularFile,
+  ChunkedFile,
+  FileVariables,
+  FileStatus,
+  ChunkedFilesInProgress
+} from './types';
+
+interface UseChunkUploadResponse {
+  addFiles: (item: RegularFile[]) => void;
+  cancelFile: (item: RegularFile['uuid']) => void;
   isLoading: boolean;
 }
 
 interface ChunkUploadOptions {
   concurrentProcess?: number;
-  onStatusUpdate: (item: UploadItem, newStatus: FileUploadStatus) => void;
+  updateFileVariables: (itemId: ChunkedFile['uuid'], variables: FileVariables) => void;
   onComplete: () => void;
 }
 
 const useChunkUpload = ({
   concurrentProcess = DEFAULT_CONCURRENT_MAX_CONNECTIONS,
-  onStatusUpdate,
+  updateFileVariables,
   onComplete
-}: ChunkUploadOptions): UseUploadQueueResponse => {
+}: ChunkUploadOptions): UseChunkUploadResponse => {
   const config = getLastKnownConfig();
   const chunkSize = config?.storage_browser?.file_upload_chunk_size ?? DEFAULT_CHUNK_SIZE;
-  const [processingItem, setProcessingItem] = useState<UploadItem>();
-  const [pendingUploadItems, setPendingUploadItems] = useState<UploadItem[]>([]);
-  const [awaitingStatusItems, setAwaitingStatusItems] = useState<UploadItem[]>([]);
-
-  const onError = () => {
-    if (processingItem) {
-      onStatusUpdate(processingItem, FileUploadStatus.Failed);
-      setProcessingItem(undefined);
-    }
-  };
-
-  const onSuccess = (item: UploadItem) => () => {
-    setAwaitingStatusItems(prev => [...prev, item]);
-    setProcessingItem(undefined);
-  };
+  const [filesWaitingFinalStatus, setFilesWaitingFinalStatus] = useState<ChunkedFile['uuid'][]>([]);
+  const [filesInProgress, setFilesInProgress] = useState<ChunkedFilesInProgress>({});
 
   const { save } = useSaveData(undefined, {
     postOptions: {
       qsEncodeData: false,
       headers: { 'Content-Type': 'multipart/form-data' }
-    },
-    onError
+    }
   });
 
-  const updateItemStatus = (serverResponse: TaskServerResponse[]) => {
-    const statusMap = getStatusHashMap(serverResponse);
-
-    const remainingItems = awaitingStatusItems.filter(item => {
-      const status = statusMap[item.uuid];
-      if (status === TaskStatus.Success || status === TaskStatus.Failure) {
-        const ItemStatus =
-          status === TaskStatus.Success ? FileUploadStatus.Uploaded : FileUploadStatus.Failed;
-        onStatusUpdate(item, ItemStatus);
-        return false;
+  const processTaskServerResponse = (response: TaskServerResponse[]) => {
+    const statusMap = getStatusHashMap(response);
+    setFilesWaitingFinalStatus(prev => {
+      const remainingFiles = prev.filter(uuid => {
+        const fileStatus = statusMap[uuid];
+        if (fileStatus === TaskStatus.Success || fileStatus === TaskStatus.Failure) {
+          const mappedStatus =
+            fileStatus === TaskStatus.Success ? FileStatus.Uploaded : FileStatus.Failed;
+          updateFileVariables(uuid, { status: mappedStatus });
+          return false; // remove the file as final status is received
+        }
+        return true;
+      });
+      if (remainingFiles.length === 0) {
+        onComplete();
       }
-      return true;
+      return remainingFiles;
     });
-    if (remainingItems.length === 0) {
-      onComplete();
-    }
-    setAwaitingStatusItems(remainingItems);
   };
 
-  const { data: tasksStatus } = useLoadData<TaskServerResponse[]>(
-    '/desktop/api2/taskserver/get_taskserver_tasks/',
-    {
-      pollInterval: awaitingStatusItems.length ? 5000 : undefined,
-      skip: !awaitingStatusItems.length,
-      transformKeys: 'none'
-    }
-  );
+  useLoadData<TaskServerResponse[]>('/desktop/api2/taskserver/get_taskserver_tasks/', {
+    pollInterval: 5000,
+    skip: filesWaitingFinalStatus.length === 0,
+    onSuccess: processTaskServerResponse,
+    transformKeys: 'none'
+  });
 
-  useEffect(() => {
-    if (tasksStatus) {
-      updateItemStatus(tasksStatus);
-    }
-  }, [tasksStatus]);
-
-  const onChunksUploadComplete = async () => {
-    if (processingItem) {
-      const { url, payload } = getChunksCompletePayload(processingItem, chunkSize);
-      return save(payload, {
-        url,
-        onSuccess: onSuccess(processingItem)
-      });
-    }
+  const handleAllChunksUploaded = (chunk: ChunkedFile) => {
+    const { url, payload } = getChunksCompletePayload(chunk);
+    return save(payload, {
+      url,
+      onSuccess: () => setFilesWaitingFinalStatus(prev => [...prev, chunk.uuid]),
+      onError: error => updateFileVariables(chunk.uuid, { status: FileStatus.Failed, error })
+    });
   };
 
-  const uploadChunk = async (chunkItem: UploadChunkItem) => {
-    const { url, payload } = getChunkItemPayload(chunkItem, chunkSize);
-    return save(payload, { url });
+  const onChunkUploadSuccess = (chunk: ChunkedFile) => () => {
+    setFilesInProgress(prev => {
+      const isAllChunksUploaded = isAllChunksOfFileUploaded(prev, chunk);
+      if (isAllChunksUploaded) {
+        handleAllChunksUploaded(chunk);
+        delete prev[chunk.uuid];
+      }
+
+      return prev;
+    });
   };
 
-  const { enqueue } = useQueueProcessor<UploadChunkItem>(uploadChunk, {
-    concurrentProcess,
-    onSuccess: onChunksUploadComplete
-  });
+  const onUploadProgress = (chunk: ChunkedFile) => (chunkProgress: ProgressEvent) => {
+    setFilesInProgress(prev => {
+      const allChunks = prev[chunk.uuid] || [];
+      const chunk1 = allChunks.find(c => c.chunkNumber === chunk.chunkNumber);
+      if (!chunk1) {
+        return prev;
+      }
+      chunk1.progress = getItemProgress(chunkProgress);
 
-  const uploadItemInChunks = (item: UploadItem) => {
-    const chunks = createChunks(item, chunkSize);
-    return enqueue(chunks);
+      const totalProgress = getItemsTotalProgress(chunk, allChunks);
+      updateFileVariables(chunk.uuid, { progress: totalProgress });
+      return { ...prev, [chunk.uuid]: allChunks };
+    });
   };
 
-  const uploadItemInSingleChunk = async (item: UploadItem) => {
-    const { url, payload } = getChunkSinglePayload(item, chunkSize);
+  const uploadChunkToServer = async (chunk: ChunkedFile) => {
+    const { url, payload } = getChunkItemPayload(chunk);
     return save(payload, {
       url,
-      onSuccess: onSuccess(item)
+      onSuccess: onChunkUploadSuccess(chunk),
+      onError: error => updateFileVariables(chunk.uuid, { status: FileStatus.Failed, error }),
+      postOptions: { onUploadProgress: onUploadProgress(chunk) }
     });
   };
 
-  const checkAvailableSpace = async (fileSize: number) => {
-    const { upload_available_space: availableSpace } = await get<{
-      upload_available_space: number;
-    }>(UPLOAD_AVAILABLE_SPACE_URL);
-    return availableSpace >= fileSize;
-  };
-
-  const uploadItem = async (item: UploadItem) => {
-    const isSpaceAvailable = await checkAvailableSpace(item.file.size);
-    if (!isSpaceAvailable) {
-      onStatusUpdate(item, FileUploadStatus.Failed);
-      return Promise.resolve();
+  const processChunkedFile = async (chunk: ChunkedFile): Promise<void> => {
+    const isFirstChunk = !filesInProgress[chunk.uuid];
+    if (isFirstChunk) {
+      updateFileVariables(chunk.uuid, { status: FileStatus.Uploading });
+      const isUploadPossible = await isSpaceAvailableInServer(chunk.totalSize);
+      if (!isUploadPossible) {
+        const error = new Error('Upload server ran out of space. Try again later.');
+        cancelFile(chunk.uuid);
+        return updateFileVariables(chunk.uuid, { status: FileStatus.Failed, error });
+      }
     }
+    setFilesInProgress(prev => addChunkToInProcess(prev, chunk));
 
-    onStatusUpdate(item, FileUploadStatus.Uploading);
-    const chunks = getTotalChunk(item.file.size, chunkSize);
-    if (chunks === 1) {
-      return uploadItemInSingleChunk(item);
-    }
-    return uploadItemInChunks(item);
+    return uploadChunkToServer(chunk);
   };
 
-  const addFiles = (newItems: UploadItem[]) => {
-    setPendingUploadItems(prev => [...prev, ...newItems]);
-  };
+  const { enqueue, dequeue } = useQueueProcessor<ChunkedFile>(processChunkedFile, {
+    concurrentProcess
+  });
 
-  const removeFile = (item: UploadItem) => {
-    setPendingUploadItems(prev => prev.filter(i => i.uuid !== item.uuid));
+  const addFiles = (newFiles: RegularFile[]) => {
+    newFiles.forEach(file => {
+      const chunks = createChunks(file, chunkSize);
+      enqueue(chunks);
+    });
   };
 
-  useEffect(() => {
-    // Ensures one file is broken down in chunks and uploaded to the server
-    if (!processingItem && pendingUploadItems.length) {
-      const item = pendingUploadItems[0];
-      setProcessingItem(item);
-      setPendingUploadItems(prev => prev.slice(1));
-      uploadItem(item);
-    }
-  }, [pendingUploadItems, processingItem]);
+  const cancelFile = (fileUuid: ChunkedFile['uuid']) => dequeue(fileUuid, 'uuid');
 
-  return { addFiles, removeFile, isLoading: !!processingItem || !!pendingUploadItems.length };
+  return {
+    addFiles,
+    cancelFile,
+    isLoading: !!(filesWaitingFinalStatus.length || filesInProgress.length)
+  };
 };
 
 export default useChunkUpload;

+ 42 - 34
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useFileUpload.ts

@@ -17,82 +17,90 @@
 import { useCallback, useEffect, useState } from 'react';
 import useRegularUpload from './useRegularUpload';
 import useChunkUpload from './useChunkUpload';
-import { getNewFileItems, UploadItem } from './util';
-import {
-  DEFAULT_CONCURRENT_MAX_CONNECTIONS,
-  FileUploadStatus
-} from '../../constants/storageBrowser';
+import { DEFAULT_CONCURRENT_MAX_CONNECTIONS } from '../../constants/storageBrowser';
 import { getLastKnownConfig } from '../../../config/hueConfig';
+import { getNewRegularFiles } from './utils';
+import { FileStatus, RegularFile, FileVariables } from './types';
 
 interface UseUploadQueueResponse {
-  uploadQueue: UploadItem[];
-  onCancel: (item: UploadItem) => void;
+  uploadQueue: RegularFile[];
+  onCancel: (item: RegularFile) => void;
   isLoading: boolean;
 }
 
 interface UploadQueueOptions {
   isChunkUpload?: boolean;
-  onComplete: () => UploadItem[] | void;
+  onComplete: () => RegularFile[] | void;
 }
 
 const useFileUpload = (
-  filesQueue: UploadItem[],
+  filesQueue: RegularFile[],
   { isChunkUpload = false, onComplete }: UploadQueueOptions
 ): UseUploadQueueResponse => {
   const config = getLastKnownConfig();
   const concurrentProcess =
     config?.storage_browser.concurrent_max_connection ?? DEFAULT_CONCURRENT_MAX_CONNECTIONS;
 
-  const [uploadQueue, setUploadQueue] = useState<UploadItem[]>([]);
-
-  const onStatusUpdate = (item: UploadItem, newStatus: FileUploadStatus) =>
-    setUploadQueue(prev =>
-      prev.map(queueItem =>
-        queueItem.uuid === item.uuid ? { ...queueItem, status: newStatus } : queueItem
-      )
-    );
-
-  const findQueueItem = (item: UploadItem) =>
-    uploadQueue.filter(queueItem => queueItem.uuid === item.uuid)?.[0];
+  const [uploadQueue, setUploadQueue] = useState<RegularFile[]>([]);
+
+  const updateFileVariables = (
+    itemId: RegularFile['uuid'],
+    { status, error, progress }: FileVariables
+  ) => {
+    setUploadQueue(prev => {
+      return prev.map(queueItem => {
+        if (queueItem.uuid === itemId) {
+          return {
+            ...queueItem,
+            status: status ?? queueItem.status,
+            error: error ?? queueItem.error,
+            progress: progress ?? queueItem.progress
+          };
+        }
+        return queueItem;
+      });
+    });
+  };
 
   const {
     addFiles: addToChunkUpload,
-    removeFile: removeFromChunkUpload,
+    cancelFile: cancelFromChunkUpload,
     isLoading: isChunkLoading
   } = useChunkUpload({
     concurrentProcess,
-    onStatusUpdate,
+    updateFileVariables,
     onComplete
   });
 
   const {
     addFiles: addToRegularUpload,
-    removeFile: removeFromRegularUpload,
-    isLoading: isNonChunkLoading
+    cancelFile: cancelFromRegularUpload,
+    isLoading: isRegularLoading
   } = useRegularUpload({
     concurrentProcess,
-    onStatusUpdate,
+    updateFileVariables,
     onComplete
   });
 
   const onCancel = useCallback(
-    (item: UploadItem) => {
-      const queueItem = findQueueItem(item);
-      if (queueItem.status === FileUploadStatus.Pending) {
-        onStatusUpdate(item, FileUploadStatus.Canceled);
+    (item: RegularFile) => {
+      const queueItem = uploadQueue.find(q => q.uuid === item.uuid);
+      if (queueItem?.status === FileStatus.Pending) {
+        const error = new Error('Upload cancelled');
+        updateFileVariables(item.uuid, { status: FileStatus.Cancelled, error });
 
         if (isChunkUpload) {
-          removeFromChunkUpload(item);
+          cancelFromChunkUpload(item.uuid);
         } else {
-          removeFromRegularUpload(item);
+          cancelFromRegularUpload(item.uuid);
         }
       }
     },
-    [isChunkUpload, onStatusUpdate, removeFromChunkUpload, removeFromRegularUpload]
+    [isChunkUpload, updateFileVariables, cancelFromChunkUpload, cancelFromRegularUpload]
   );
 
   useEffect(() => {
-    const newQueueItems = getNewFileItems(filesQueue, uploadQueue);
+    const newQueueItems = getNewRegularFiles(filesQueue, uploadQueue);
 
     if (newQueueItems.length > 0) {
       setUploadQueue(prev => [...prev, ...newQueueItems]);
@@ -105,7 +113,7 @@ const useFileUpload = (
     }
   }, [filesQueue, uploadQueue, isChunkUpload]);
 
-  return { uploadQueue, onCancel, isLoading: isChunkLoading || isNonChunkLoading };
+  return { uploadQueue, onCancel, isLoading: isChunkLoading || isRegularLoading };
 };
 
 export default useFileUpload;

+ 23 - 17
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useRegularUpload.ts

@@ -16,28 +16,26 @@
 
 import useQueueProcessor from '../useQueueProcessor/useQueueProcessor';
 import { UPLOAD_FILE_URL } from '../../../apps/storageBrowser/api';
-import {
-  DEFAULT_CONCURRENT_MAX_CONNECTIONS,
-  FileUploadStatus
-} from '../../constants/storageBrowser';
+import { DEFAULT_CONCURRENT_MAX_CONNECTIONS } from '../../constants/storageBrowser';
 import useSaveData from '../useSaveData/useSaveData';
-import { UploadItem } from './util';
+import { getItemProgress } from './utils';
+import { RegularFile, FileVariables, FileStatus } from './types';
 
 interface UseUploadQueueResponse {
-  addFiles: (item: UploadItem[]) => void;
-  removeFile: (item: UploadItem) => void;
+  addFiles: (item: RegularFile[]) => void;
+  cancelFile: (uuid: RegularFile['uuid']) => void;
   isLoading: boolean;
 }
 
 interface UploadQueueOptions {
   concurrentProcess?: number;
-  onStatusUpdate: (item: UploadItem, newStatus: FileUploadStatus) => void;
+  updateFileVariables: (item: RegularFile['uuid'], variables: FileVariables) => void;
   onComplete: () => void;
 }
 
 const useRegularUpload = ({
   concurrentProcess = DEFAULT_CONCURRENT_MAX_CONNECTIONS,
-  onStatusUpdate,
+  updateFileVariables,
   onComplete
 }: UploadQueueOptions): UseUploadQueueResponse => {
   const { save } = useSaveData(UPLOAD_FILE_URL, {
@@ -49,8 +47,8 @@ const useRegularUpload = ({
     }
   });
 
-  const processUploadItem = async (item: UploadItem) => {
-    onStatusUpdate(item, FileUploadStatus.Uploading);
+  const processRegularFile = async (item: RegularFile) => {
+    updateFileVariables(item.uuid, { status: FileStatus.Uploading });
 
     const payload = new FormData();
     payload.append('file', item.file);
@@ -58,24 +56,32 @@ const useRegularUpload = ({
 
     return save(payload, {
       onSuccess: () => {
-        onStatusUpdate(item, FileUploadStatus.Uploaded);
+        updateFileVariables(item.uuid, { status: FileStatus.Uploaded });
       },
-      onError: () => {
-        onStatusUpdate(item, FileUploadStatus.Failed);
+      onError: error => {
+        updateFileVariables(item.uuid, { status: FileStatus.Failed, error });
+      },
+      postOptions: {
+        onUploadProgress: progress => {
+          const itemProgress = getItemProgress(progress);
+          updateFileVariables(item.uuid, { progress: itemProgress });
+        }
       }
     });
   };
 
   const {
     enqueue: addFiles,
-    dequeue: removeFile,
+    dequeue,
     isLoading
-  } = useQueueProcessor<UploadItem>(processUploadItem, {
+  } = useQueueProcessor<RegularFile>(processRegularFile, {
     concurrentProcess,
     onSuccess: onComplete
   });
 
-  return { addFiles, removeFile, isLoading };
+  const cancelFile = (itemId: RegularFile['uuid']) => dequeue(itemId, 'uuid');
+
+  return { addFiles, cancelFile, isLoading };
 };
 
 export default useRegularUpload;

+ 0 - 137
desktop/core/src/desktop/js/utils/hooks/useFileUpload/util.ts

@@ -1,137 +0,0 @@
-// Licensed to Cloudera, Inc. under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  Cloudera, Inc. licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { FileUploadStatus } from 'utils/constants/storageBrowser';
-import { TaskServerResponse, TaskStatus } from '../../../reactComponents/TaskBrowser/TaskBrowser';
-import { CHUNK_UPLOAD_URL, CHUNK_UPLOAD_COMPLETE_URL } from '../../../apps/storageBrowser/api';
-
-export interface UploadItem {
-  uuid: string;
-  filePath: string;
-  file: File;
-  status: FileUploadStatus;
-}
-
-export interface UploadMetaData {
-  qqtotalparts: string;
-  qqtotalfilesize: string;
-  qqfilename: string;
-  inputName: string;
-  dest: string;
-  qquuid: string;
-}
-
-export interface UploadChunkItem extends UploadItem {
-  chunkIndex: number;
-}
-
-interface ChunkPayload {
-  url: string;
-  payload: FormData;
-}
-
-export const getNewFileItems = (newQueue: UploadItem[], oldQueue: UploadItem[]): UploadItem[] => {
-  return newQueue.filter(
-    newItem =>
-      !oldQueue.some(
-        oldItem => oldItem.file.name === newItem.file.name && oldItem.filePath === newItem.filePath
-      )
-  );
-};
-
-export const getTotalChunk = (fileSize: number, DEFAULT_CHUNK_SIZE: number): number => {
-  return Math.ceil(fileSize / DEFAULT_CHUNK_SIZE);
-};
-
-export const getMetaData = (item: UploadItem, DEFAULT_CHUNK_SIZE: number): UploadMetaData => ({
-  qqtotalparts: String(getTotalChunk(item.file.size, DEFAULT_CHUNK_SIZE)),
-  qqtotalfilesize: String(item.file.size),
-  qqfilename: item.file.name,
-  inputName: 'hdfs_file',
-  dest: item.filePath,
-  qquuid: item.uuid
-});
-
-export const createChunks = (item: UploadItem, chunkSize: number): UploadChunkItem[] => {
-  const totalChunks = getTotalChunk(item.file.size, chunkSize);
-
-  const chunks = Array.from({ length: totalChunks }, (_, i) => ({
-    ...item,
-    chunkIndex: i
-  }));
-
-  return chunks;
-};
-
-export const getStatusHashMap = (
-  serverResponse: TaskServerResponse[]
-): Record<string, TaskStatus> =>
-  serverResponse.reduce(
-    (acc, row: TaskServerResponse) => ({
-      ...acc,
-      [row.task_id]: row.status
-    }),
-    {}
-  );
-
-export const getChunkItemPayload = (
-  chunkItem: UploadChunkItem,
-  chunkSize: number
-): ChunkPayload => {
-  const chunkStart = (chunkItem.chunkIndex ?? 0) * chunkSize;
-  const chunkEnd = Math.min(chunkStart + chunkSize, chunkItem!.file.size);
-
-  const metaData = getMetaData(chunkItem, chunkSize);
-  const chunkQueryParams = new URLSearchParams({
-    ...metaData,
-    qqpartindex: String(chunkItem.chunkIndex),
-    qqpartbyteoffset: String(chunkStart),
-    qqchunksize: String(chunkEnd - chunkStart)
-  }).toString();
-  const url = `${CHUNK_UPLOAD_URL}?${chunkQueryParams}`;
-
-  const payload = new FormData();
-  payload.append('hdfs_file', chunkItem!.file.slice(chunkStart, chunkEnd));
-  return { url, payload };
-};
-
-export const getChunksCompletePayload = (
-  processingItem: UploadItem,
-  chunkSize: number
-): ChunkPayload => {
-  const fileMetaData = getMetaData(processingItem, chunkSize);
-  const payload = new FormData();
-  Object.entries(fileMetaData).forEach(([key, value]) => {
-    payload.append(key, value);
-  });
-  return { url: CHUNK_UPLOAD_COMPLETE_URL, payload };
-};
-
-export const getChunkSinglePayload = (item: UploadItem, chunkSize: number): ChunkPayload => {
-  const metaData = getMetaData(item, chunkSize);
-
-  const singleChunkParams = Object.fromEntries(
-    Object.entries(metaData).filter(([key]) => key !== 'qqtotalparts')
-  );
-
-  const queryParams = new URLSearchParams(singleChunkParams).toString();
-  const url = `${CHUNK_UPLOAD_URL}?${queryParams}`;
-
-  const payload = new FormData();
-  payload.append('hdfs_file', item.file);
-
-  return { url, payload };
-};

+ 166 - 0
desktop/core/src/desktop/js/utils/hooks/useFileUpload/utils.ts

@@ -0,0 +1,166 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { get } from '../../../api/utils';
+import {
+  CHUNK_UPLOAD_URL,
+  CHUNK_UPLOAD_COMPLETE_URL,
+  UPLOAD_AVAILABLE_SPACE_URL
+} from '../../../apps/storageBrowser/api';
+import { TaskServerResponse, TaskStatus } from '../../../reactComponents/TaskBrowser/TaskBrowser';
+import {
+  ChunkedFile,
+  ChunkedFilesInProgress,
+  FileChunkMetaData,
+  FileUploadApiPayload,
+  RegularFile
+} from './types';
+
+export const getNewRegularFiles = (
+  newQueue: RegularFile[],
+  oldQueue: RegularFile[]
+): RegularFile[] => {
+  return newQueue.filter(
+    newItem =>
+      !oldQueue.some(
+        oldItem => oldItem.file.name === newItem.file.name && oldItem.filePath === newItem.filePath
+      )
+  );
+};
+
+export const getTotalChunk = (fileSize: number, DEFAULT_CHUNK_SIZE: number): number => {
+  return Math.ceil(fileSize / DEFAULT_CHUNK_SIZE);
+};
+
+export const getMetaData = (item: ChunkedFile): FileChunkMetaData => ({
+  qqtotalparts: String(item.totalChunks),
+  qqtotalfilesize: String(item.totalSize),
+  qqfilename: item.fileName,
+  inputName: 'hdfs_file',
+  dest: item.filePath,
+  qquuid: item.uuid
+});
+
+export const createChunks = (item: RegularFile, chunkSize: number): ChunkedFile[] => {
+  const totalChunks = getTotalChunk(item.file.size, chunkSize);
+
+  const chunks = Array.from({ length: totalChunks }, (_, i) => {
+    const chunkStartOffset = i * chunkSize;
+    const chunkEndOffset = Math.min(chunkStartOffset + chunkSize, item.file.size);
+    return {
+      ...item,
+      fileName: item.file.name,
+      totalSize: item.file.size,
+      file: item.file.slice(chunkStartOffset, chunkEndOffset),
+      totalChunks,
+      chunkNumber: i,
+      chunkStartOffset,
+      chunkEndOffset
+    };
+  });
+
+  return chunks;
+};
+
+export const getStatusHashMap = (
+  serverResponse: TaskServerResponse[]
+): Record<string, TaskStatus> =>
+  serverResponse.reduce(
+    (acc, row: TaskServerResponse) => ({
+      ...acc,
+      [row.task_id]: row.status
+    }),
+    {}
+  );
+
+export const getChunkItemPayload = (chunkItem: ChunkedFile): FileUploadApiPayload => {
+  const metaData = getMetaData(chunkItem);
+  const chunkQueryParams = new URLSearchParams({
+    ...metaData,
+    qqpartindex: String(chunkItem.chunkNumber),
+    qqpartbyteoffset: String(chunkItem.chunkStartOffset),
+    qqchunksize: String(chunkItem.chunkEndOffset - chunkItem.chunkStartOffset)
+  }).toString();
+
+  const url = `${CHUNK_UPLOAD_URL}?${chunkQueryParams}`;
+
+  const payload = new FormData();
+  payload.append('hdfs_file', chunkItem.file);
+  return { url, payload };
+};
+
+export const getChunksCompletePayload = (processingItem: ChunkedFile): FileUploadApiPayload => {
+  const fileMetaData = getMetaData(processingItem);
+  const payload = new FormData();
+  Object.entries(fileMetaData).forEach(([key, value]) => {
+    payload.append(key, value);
+  });
+  return { url: CHUNK_UPLOAD_COMPLETE_URL, payload };
+};
+
+export const getItemProgress = (progress?: ProgressEvent): number => {
+  if (!progress?.total || !progress?.loaded || progress?.total === 0) {
+    return 0;
+  }
+  return Math.round((progress.loaded * 100) / progress.total);
+};
+
+export const getItemsTotalProgress = (
+  chunkItem?: ChunkedFile,
+  chunks?: ChunkedFilesInProgress['uuid']
+): number => {
+  if (!chunkItem || !chunks) {
+    return 0;
+  }
+  return chunks.reduce((acc, chunk) => {
+    return acc + (chunk.progress * chunk.chunkSize) / chunkItem.totalSize;
+  }, 0);
+};
+
+export const addChunkToInProcess = (
+  currentInProcess: ChunkedFilesInProgress,
+  chunkItem: ChunkedFile
+): ChunkedFilesInProgress => {
+  const inProcessChunkObj = {
+    chunkNumber: chunkItem.chunkNumber,
+    progress: 0,
+    chunkSize: chunkItem.file.size
+  };
+  if (currentInProcess[chunkItem.uuid] === undefined) {
+    currentInProcess[chunkItem.uuid] = [inProcessChunkObj];
+  } else {
+    currentInProcess[chunkItem.uuid].push(inProcessChunkObj);
+  }
+  return currentInProcess;
+};
+
+export const isSpaceAvailableInServer = async (fileSize: number): Promise<boolean> => {
+  const response = await get<{
+    upload_available_space: number;
+  }>(UPLOAD_AVAILABLE_SPACE_URL);
+  return !!response?.upload_available_space && response.upload_available_space >= fileSize;
+};
+
+export const isAllChunksOfFileUploaded = (
+  filesInProgress: ChunkedFilesInProgress,
+  chunk: ChunkedFile
+): boolean => {
+  const fileAllChunks = filesInProgress[chunk.uuid];
+  const isTotalChunkCountMatched = fileAllChunks.length === chunk.totalChunks;
+  const isAllChunksUploaded = filesInProgress[chunk.uuid]?.every(chunk => chunk.progress === 100);
+
+  return isTotalChunkCountMatched && isAllChunksUploaded;
+};

+ 26 - 0
desktop/core/src/desktop/js/utils/hooks/useQueueProcessor/useQueueProcessor.test.tsx

@@ -99,6 +99,32 @@ describe('useQueueProcessor', () => {
     expect(result.current.queue).toEqual(['item3']);
   });
 
+  it('should update the queue when an item is dequeued with itemKey', async () => {
+    const { result } = renderHook(() =>
+      useQueueProcessor<{ id: number; name: string }>(mockProcessItem, { concurrentProcess: 1 })
+    );
+
+    act(() => {
+      result.current.enqueue([
+        { id: 1, name: 'item1' },
+        { id: 2, name: 'item2' },
+        { id: 3, name: 'item3' }
+      ]);
+    });
+
+    expect(result.current.queue).toEqual([
+      { id: 2, name: 'item2' },
+      { id: 3, name: 'item3' }
+    ]);
+    expect(result.current.isLoading).toBe(true);
+
+    act(() => {
+      result.current.dequeue('item2', 'name');
+    });
+
+    expect(result.current.queue).toEqual([{ id: 3, name: 'item3' }]);
+  });
+
   it('should update isLoading when items are being processed', async () => {
     const { result } = renderHook(() =>
       useQueueProcessor(mockProcessItem, { concurrentProcess: 1 })

+ 7 - 16
desktop/core/src/desktop/js/utils/hooks/useQueueProcessor/useQueueProcessor.ts

@@ -19,8 +19,7 @@ import { useState, useEffect } from 'react';
 interface UseQueueProcessorResult<T> {
   queue: T[];
   enqueue: (newItems: T[]) => void;
-  enqueueAsync: (newItems: T[]) => Promise<void>;
-  dequeue: (item: T) => void;
+  dequeue: (itemValue: T[keyof T] | T, itemKey?: keyof T) => void;
   isLoading: boolean;
 }
 
@@ -41,22 +40,15 @@ const useQueueProcessor = <T>(
     setQueue(prevQueue => [...prevQueue, ...newItems]);
   };
 
-  const enqueueAsync = (newItems: T[]) => {
-    enqueue(newItems);
-    return new Promise<void>(resolve => {
-      const interval = setInterval(() => {
-        if (queue.length === 0 && processingQueue.length === 0) {
-          clearInterval(interval);
-          resolve();
-        }
-      }, 100);
+  const dequeue = (itemValue: T[keyof T] | T, itemKey?: keyof T) => {
+    setQueue(prev => {
+      if (itemKey) {
+        return prev.filter(i => i[itemKey] !== itemValue);
+      }
+      return prev.filter(i => i !== itemValue);
     });
   };
 
-  const dequeue = (item: T) => {
-    setQueue(prev => prev.filter(i => i !== item));
-  };
-
   const processQueueItem = async (item: T) => {
     if (!isLoading) {
       setIsLoading(true);
@@ -85,7 +77,6 @@ const useQueueProcessor = <T>(
     queue,
     isLoading,
     enqueue,
-    enqueueAsync,
     dequeue
   };
 };

+ 9 - 8
desktop/core/src/desktop/js/utils/hooks/useSaveData/useSaveData.ts

@@ -14,17 +14,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { useCallback, useEffect, useMemo, useState } from 'react';
+import { useCallback, useEffect, useState } from 'react';
 import { ApiFetchOptions, post } from '../../../api/utils';
 
 interface saveOptions<T> {
   url?: string;
   onSuccess?: (data: T) => void;
   onError?: (error: Error) => void;
+  postOptions?: ApiFetchOptions<T>;
 }
 
 export interface Options<T> extends saveOptions<T> {
-  postOptions?: ApiFetchOptions<T>;
   skip?: boolean;
 }
 
@@ -46,11 +46,6 @@ const useSaveData = <T, U = unknown>(url?: string, options?: Options<T>): UseSav
     ignoreSuccessErrors: true
   };
 
-  const postOptions = useMemo(
-    () => ({ ...postOptionsDefault, ...localOptions?.postOptions }),
-    [localOptions]
-  );
-
   const saveData = useCallback(
     async (body: U, saveOptions?: saveOptions<T>) => {
       // Avoid Posting data if the skip option is true
@@ -62,6 +57,12 @@ const useSaveData = <T, U = unknown>(url?: string, options?: Options<T>): UseSav
       setLoading(true);
       setError(undefined);
 
+      const postOptions = {
+        ...postOptionsDefault,
+        ...localOptions?.postOptions,
+        ...saveOptions?.postOptions
+      };
+
       try {
         const response = await post<T, U>(apiUrl, body, postOptions);
         setData(response);
@@ -83,7 +84,7 @@ const useSaveData = <T, U = unknown>(url?: string, options?: Options<T>): UseSav
         setLoading(false);
       }
     },
-    [url, localOptions, postOptions]
+    [url, localOptions]
   );
 
   useEffect(() => {