Преглед на файлове

[ui-sb] Changes for skip/overwrite files in filebrowser (#4143)

* Changes for skip/overwrite files in filebrowser

* Changes for skip/overwrite files in filebrowser

* Changes

* Changes 2

* Changes to support file upload while file already in queue

* Fixing lints

* Code optimisation

* Fixes

* Fixes ..

* Changes

* Changes

* Fixing lint errors

* Addressing review comments

* Updated changes

* Update FileUploadQueue.scss

* Changes after upload ui fix

* Changes after upload ui fix.

* prettier fixes

* lint fixes

* syntax fix

* test side fixes

* Addressing PR comments

* prettier fix

* Modifying unit test

* Addressing PR comments.

* Update FileUploadQueue.tsx

* Update FileUploadQueue.tsx

* Updated code

* Updated code.

* Updated js lint.

* Updated js lint..
sanyam43 преди 3 месеца
родител
ревизия
15e214ae11

+ 40 - 0
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.scss

@@ -57,3 +57,43 @@
     gap: 16px;
   }
 }
+
+.hue-conflict-detect-modal {
+  height: 50vh;
+
+  &__body {
+    display: flex;
+    flex-direction: column;
+    gap: 8px;
+  }
+}
+
+.conflict-files__container {
+  margin-top: 8px;
+  overflow-y: auto;
+  max-height: calc(50vh - 100px);
+  border-top: 1px solid vars.$fluidx-gray-400;
+  padding-top: 8px;
+}
+
+.conflict-files__item {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+}
+
+.conflict-files__icon {
+  flex-shrink: 0;
+  width: 20px;
+  height: 20px;
+
+  svg {
+    width: 100%;
+    height: 100%;
+  }
+}
+
+.conflict-files__name {
+  word-break: break-word;
+  max-width: calc(100% - 28px);
+}

+ 28 - 16
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.test.tsx

@@ -15,7 +15,7 @@
 // limitations under the License.
 
 import React from 'react';
-import { render, fireEvent } from '@testing-library/react';
+import { render, fireEvent, waitFor } from '@testing-library/react';
 import '@testing-library/jest-dom';
 import FileUploadQueue from './FileUploadQueue';
 import { FileStatus, RegularFile } from '../../utils/hooks/useFileUpload/types';
@@ -23,6 +23,11 @@ import { act } from 'react-dom/test-utils';
 import huePubSub from '../../utils/huePubSub';
 import { FILE_UPLOAD_START_EVENT } from './event';
 
+jest.mock('../../api/utils', () => ({
+  __esModule: true,
+  get: jest.fn(() => Promise.reject({ response: { status: 404 } }))
+}));
+
 const mockFilesQueue: RegularFile[] = [
   {
     uuid: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx1',
@@ -42,29 +47,36 @@ jest.mock('../../utils/hooks/useFileUpload/useFileUpload', () => ({
   __esModule: true,
   default: jest.fn(() => ({
     uploadQueue: mockFilesQueue,
-    onCancel: jest.fn(),
-    addFiles: jest.fn()
+    cancelFile: jest.fn(),
+    addFiles: jest.fn(() => {}),
+    isLoading: false
   }))
 }));
 
 describe('FileUploadQueue', () => {
-  it('should render the component with initial files in the queue', () => {
-    const { getByText } = render(<FileUploadQueue />);
+  it('should render the component with initial files in the queue', async () => {
+    const { findByText } = render(<FileUploadQueue />);
 
-    act(() => huePubSub.publish(FILE_UPLOAD_START_EVENT, { files: mockFilesQueue }));
+    act(() => {
+      huePubSub.publish(FILE_UPLOAD_START_EVENT, { files: mockFilesQueue });
+    });
 
-    expect(getByText('file1.txt')).toBeInTheDocument();
-    expect(getByText('file2.txt')).toBeInTheDocument();
+    expect(await findByText('file1.txt')).toBeInTheDocument();
+    expect(await findByText('file2.txt')).toBeInTheDocument();
   });
 
-  it('should toggle the visibility of the queue when the header is clicked', () => {
-    const { getByText, getByTestId, queryByText } = render(<FileUploadQueue />);
+  it('should toggle the visibility of the queue when the header is clicked', async () => {
+    const { getByText, queryByText, getByRole } = render(<FileUploadQueue />);
+
+    act(() => {
+      huePubSub.publish(FILE_UPLOAD_START_EVENT, { files: mockFilesQueue });
+    });
 
-    act(() => huePubSub.publish(FILE_UPLOAD_START_EVENT, { files: mockFilesQueue }));
+    // Wait for items to appear
+    await waitFor(() => expect(getByText('file1.txt')).toBeVisible());
+    await waitFor(() => expect(getByText('file2.txt')).toBeVisible());
 
-    const header = getByTestId('hue-upload-queue-container__expand-button');
-    expect(getByText('file1.txt')).toBeVisible();
-    expect(getByText('file2.txt')).toBeVisible();
+    const header = getByRole('button', { name: /toggle upload queue/i });
 
     fireEvent.click(header);
 
@@ -73,7 +85,7 @@ describe('FileUploadQueue', () => {
 
     fireEvent.click(header);
 
-    expect(getByText('file1.txt')).toBeVisible();
-    expect(getByText('file2.txt')).toBeVisible();
+    await waitFor(() => expect(getByText('file1.txt')).toBeVisible());
+    await waitFor(() => expect(getByText('file2.txt')).toBeVisible());
   });
 });

+ 159 - 46
desktop/core/src/desktop/js/reactComponents/FileUploadQueue/FileUploadQueue.tsx

@@ -14,27 +14,29 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import React, { useState } from 'react';
+import React, { useEffect, useState } from 'react';
+import FileIcon from '@cloudera/cuix-core/icons/react/DocumentationIcon';
 import CloseIcon from '@cloudera/cuix-core/icons/react/CloseIcon';
 import CaratDownIcon from '@cloudera/cuix-core/icons/react/CaratDownIcon';
+import { BorderlessButton } from 'cuix/dist/components/Button';
 import CaratUpIcon from '@cloudera/cuix-core/icons/react/CaratUpIcon';
+import Modal from 'cuix/dist/components/Modal';
 import { i18nReact } from '../../utils/i18nReact';
 import { RegularFile, FileStatus } from '../../utils/hooks/useFileUpload/types';
 import useFileUpload from '../../utils/hooks/useFileUpload/useFileUpload';
 import { DEFAULT_ENABLE_CHUNK_UPLOAD } from '../../utils/constants/storageBrowser';
+import { get } from '../../api/utils';
 import { getLastKnownConfig } from '../../config/hueConfig';
 import FileUploadRow from './FileUploadRow/FileUploadRow';
 import { useHuePubSub } from '../../utils/hooks/useHuePubSub/useHuePubSub';
 import huePubSub from '../../utils/huePubSub';
-import { BorderlessButton } from 'cuix/dist/components/Button';
 import { FILE_UPLOAD_START_EVENT, FILE_UPLOAD_SUCCESS_EVENT } from './event';
-
+import { FILE_STATS_API_URL } from '../../apps/storageBrowser/api';
 import './FileUploadQueue.scss';
 
 interface FileUploadEvent {
   files: RegularFile[];
 }
-
 const sortOrder = [
   FileStatus.Uploading,
   FileStatus.Failed,
@@ -45,12 +47,28 @@ const sortOrder = [
   acc[status] = index + 1;
   return acc;
 }, {});
-
 const FileUploadQueue = (): JSX.Element => {
   const { t } = i18nReact.useTranslation();
-
   const [expandQueue, setExpandQueue] = useState<boolean>(true);
   const [isVisible, setIsVisible] = useState<boolean>(false);
+  const [conflictingFiles, setConflictingFiles] = useState<RegularFile[]>([]);
+
+  // TODO: Need to change this function with a new endpoint once available.
+  const checkFileExists = async (filePath: string): Promise<boolean> => {
+    try {
+      const response = await get<{ path: string }>(
+        FILE_STATS_API_URL,
+        { path: filePath },
+        { silenceErrors: true }
+      );
+      return response?.path === filePath;
+    } catch (error) {
+      if (error.response?.status === 404) {
+        return false;
+      }
+      return false;
+    }
+  };
 
   const onComplete = () => {
     huePubSub.publish(FILE_UPLOAD_SUCCESS_EVENT);
@@ -66,28 +84,98 @@ const FileUploadQueue = (): JSX.Element => {
     onComplete
   });
 
+  useEffect(() => {
+    if (uploadQueue.length > 0) {
+      setIsVisible(true);
+    }
+  }, [uploadQueue.length]);
+
   useHuePubSub<FileUploadEvent>({
     topic: FILE_UPLOAD_START_EVENT,
-    callback: (data?: FileUploadEvent) => {
-      if (data?.files) {
-        setIsVisible(true);
-        addFiles(data.files);
+    callback: async (data?: FileUploadEvent) => {
+      const newFiles = data?.files ?? [];
+      if (newFiles.length === 0) {
+        huePubSub.publish('hue.global.error', { message: 'Something went wrong!' });
+        return;
+      }
+      if (newFiles.length > 0) {
+        const { conflicts, nonConflictingFiles } = await detectFileConflicts(newFiles, uploadQueue);
+        if (conflicts.length > 0) {
+          setConflictingFiles(conflicts);
+        } else {
+          setConflictingFiles([]);
+        }
+        if (nonConflictingFiles.length > 0) {
+          addFiles(nonConflictingFiles);
+          setIsVisible(true);
+        }
       }
     }
   });
 
+  const detectFileConflicts = async (
+    newFiles: RegularFile[],
+    uploadQueue: RegularFile[]
+  ): Promise<{ conflicts: RegularFile[]; nonConflictingFiles: RegularFile[] }> => {
+    const inProgressFileIdentifiers = new Set(
+      uploadQueue
+        .filter(file => file.status === FileStatus.Uploading || file.status === FileStatus.Pending)
+        .map(file => `${file.filePath}/${file.file.name}`)
+    );
+
+    const result = await newFiles.reduce<
+      Promise<{ conflicts: RegularFile[]; nonConflictingFiles: RegularFile[] }>
+    >(
+      async (accPromise, newFile) => {
+        const acc = await accPromise;
+        const fullFilePath = `${newFile.filePath}/${newFile.file.name}`;
+
+        if (inProgressFileIdentifiers.has(fullFilePath)) {
+          return acc;
+        }
+
+        const exists = await checkFileExists(fullFilePath);
+        if (exists) {
+          return {
+            ...acc,
+            conflicts: [...acc.conflicts, newFile]
+          };
+        } else {
+          return {
+            ...acc,
+            nonConflictingFiles: [...acc.nonConflictingFiles, newFile]
+          };
+        }
+      },
+      Promise.resolve({ conflicts: [], nonConflictingFiles: [] })
+    );
+
+    return result;
+  };
+
   const onClose = () => {
     uploadQueue.forEach(file => cancelFile(file));
     setIsVisible(false);
   };
 
+  const handleConflictResolution = (overwrite: boolean) => {
+    if (overwrite) {
+      const updatedFilesWithOverwriteFlag = conflictingFiles.map(file => ({
+        ...file,
+        overwrite: true
+      }));
+      addFiles(updatedFilesWithOverwriteFlag);
+    }
+    setConflictingFiles([]);
+    setIsVisible(true);
+  };
+
   const uploadedCount = uploadQueue.filter(item => item.status === FileStatus.Uploaded).length;
   const pendingCount = uploadQueue.filter(
     item => item.status === FileStatus.Pending || item.status === FileStatus.Uploading
   ).length;
   const failedCount = uploadQueue.filter(item => item.status === FileStatus.Failed).length;
-
-  if (!isVisible) {
+  if (!isVisible && conflictingFiles.length === 0) {
     return <></>;
   }
 
@@ -101,43 +189,68 @@ const FileUploadQueue = (): JSX.Element => {
     }
     return `${uploadedText}${failedText}`;
   };
-
   return (
-    <div className="hue-upload-queue-container antd cuix">
-      <div className="hue-upload-queue-container__header" data-testid="hue-upload-queue__header">
-        {t(getHeaderText(), {
-          pendingCount,
-          uploadedCount,
-          failedCount
-        })}
-        <div className="hue-upload-queue-container__header__button-group">
-          <BorderlessButton
-            onClick={() => setExpandQueue(!expandQueue)}
-            data-testid="hue-upload-queue-container__expand-button"
-            icon={expandQueue ? <CaratDownIcon /> : <CaratUpIcon />}
-          />
-          <BorderlessButton
-            onClick={onClose}
-            data-testid="hue-upload-queue-container__close-button"
-            icon={<CloseIcon />}
-          />
-        </div>
-      </div>
-      {expandQueue && (
-        <div className="hue-upload-queue-container__list">
-          {uploadQueue
-            .sort((a, b) => sortOrder[a.status] - sortOrder[b.status])
-            .map((row: RegularFile) => (
-              <FileUploadRow
-                key={`${row.filePath}__${row.file.name}`}
-                data={row}
-                onCancel={() => cancelFile(row)}
-              />
+    <>
+      {conflictingFiles.length > 0 && (
+        <Modal
+          title={t('Detected Filename Conflicts')}
+          open={true}
+          okText={t('Overwrite')}
+          onOk={() => handleConflictResolution(true)}
+          cancelText={t('Cancel')}
+          onCancel={() => setConflictingFiles([])}
+          secondaryButtonText={t('Skip Upload')}
+          onSecondary={() => handleConflictResolution(false)}
+          className="hue-conflict-detect-modal"
+        >
+          {(() => {
+            const count = conflictingFiles.length;
+            const noun = count === 1 ? 'file' : 'files';
+            const verb = count === 1 ? 'exists' : 'exist';
+            return t(
+              `${count} ${noun} you are trying to upload already ${verb} in the uploaded files.`
+            );
+          })()}
+          <div className="conflict-files__container">
+            {conflictingFiles.map(file => (
+              <div key={file.file.name} className="conflict-files__item">
+                <div className="conflict-files__icon">
+                  <FileIcon />
+                </div>
+                <span className="conflict-files__name">{file.file.name}</span>
+              </div>
             ))}
-        </div>
+          </div>
+        </Modal>
       )}
-    </div>
+      <div className="hue-upload-queue-container antd cuix">
+        <div className="hue-upload-queue-container__header">
+          {t(getHeaderText(), { pendingCount, uploadedCount, failedCount })}
+          <div className="hue-upload-queue-container__header__button-group">
+            <BorderlessButton
+              onClick={() => setExpandQueue(!expandQueue)}
+              icon={expandQueue ? <CaratDownIcon /> : <CaratUpIcon />}
+              aria-label={t('Toggle upload queue')}
+              data-testid="hue-upload-queue-container__expand-button"
+            />
+            <BorderlessButton onClick={onClose} icon={<CloseIcon />} />
+          </div>
+        </div>
+        {expandQueue && (
+          <div className="hue-upload-queue-container__list">
+            {uploadQueue
+              .sort((a, b) => sortOrder[a.status] - sortOrder[b.status])
+              .map((row: RegularFile, index: number) => (
+                <FileUploadRow
+                  key={`${row.filePath}__${row.file.name}__${index}`}
+                  data={row}
+                  onCancel={() => cancelFile(row)}
+                />
+              ))}
+          </div>
+        )}
+      </div>
+    </>
   );
 };
-
 export default FileUploadQueue;

+ 1 - 0
desktop/core/src/desktop/js/utils/hooks/useFileUpload/types.ts

@@ -30,6 +30,7 @@ export interface RegularFile {
   status: FileStatus;
   progress?: number;
   error?: Error;
+  overwrite?: boolean;
 }
 
 // Interface for file upload in chunks.

+ 1 - 1
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useChunkUpload.ts

@@ -45,7 +45,7 @@ import {
 } from './types';
 
 interface UseChunkUploadResponse {
-  addFiles: (item: RegularFile[]) => void;
+  addFiles: (items: RegularFile[]) => void;
   cancelFile: (item: RegularFile['uuid']) => void;
   isLoading: boolean;
 }

+ 49 - 15
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useRegularUpload.test.tsx

@@ -90,7 +90,6 @@ describe('useRegularUpload', () => {
 
     expect(mockDequeue).toHaveBeenCalledWith(mockFile.uuid, 'uuid');
   });
-
   it('should call save with correct payload and update file variables on success', async () => {
     const { result } = renderHook(() =>
       useRegularUpload({
@@ -100,29 +99,64 @@ describe('useRegularUpload', () => {
     );
 
     result.current.addFiles([mockFile]);
-
     if (mockProcessCallback) {
       await mockProcessCallback(mockFile);
     }
 
-    const mockFormData = new FormData();
-    mockFormData.append('file', mockFile.file);
-
-    expect(mockSave).toHaveBeenCalledWith(mockFormData, {
-      onSuccess: expect.any(Function),
-      onError: expect.any(Function),
-      postOptions: {
-        onUploadProgress: expect.any(Function),
-        params: {
-          destination_path: mockFile.filePath
-        }
-      }
+    expect(mockSave).toHaveBeenCalledTimes(1);
+    const [formDataArg, optionsArg] = mockSave.mock.calls[0];
+
+    // FormData: only the file is sent
+    expect(formDataArg).toBeInstanceOf(FormData);
+    const fdFile = formDataArg.get('file') as File;
+    expect(fdFile).toBeInstanceOf(File);
+    expect(fdFile.name).toBe('file.txt');
+    expect(formDataArg.has('destination_path')).toBe(false);
+    expect(formDataArg.has('overwrite')).toBe(false);
+
+    // Params: destination_path and overwrite=false
+    expect(optionsArg).toBeDefined();
+    expect(optionsArg.postOptions).toBeDefined();
+    expect(optionsArg.postOptions.params).toEqual({
+      destination_path: '/uploads/',
+      overwrite: 'false'
     });
+    expect(typeof optionsArg.onSuccess).toBe('function');
+    expect(typeof optionsArg.onError).toBe('function');
+    expect(typeof optionsArg.postOptions.onUploadProgress).toBe('function');
+
     expect(mockUpdateFileVariables).toHaveBeenCalledWith(mockFile.uuid, {
       status: FileStatus.Uploading
     });
+  });
+
+  it('should include overwrite=true in params when file has overwrite flag', async () => {
+    const { result } = renderHook(() =>
+      useRegularUpload({
+        updateFileVariables: mockUpdateFileVariables,
+        onComplete: mockOnComplete
+      })
+    );
 
-    expect(mockOnComplete).not.toHaveBeenCalled();
+    const fileWithOverwrite: RegularFile = { ...mockFile, overwrite: true };
+    result.current.addFiles([fileWithOverwrite]);
+    if (mockProcessCallback) {
+      await mockProcessCallback(fileWithOverwrite);
+    }
+
+    const [formDataArg, optionsArg] = mockSave.mock.calls.at(-1)!;
+
+    // FormData still only has the file
+    expect(formDataArg).toBeInstanceOf(FormData);
+    expect((formDataArg.get('file') as File).name).toBe('file.txt');
+    expect(formDataArg.has('destination_path')).toBe(false);
+    expect(formDataArg.has('overwrite')).toBe(false);
+
+    // Params: destination_path and overwrite=true
+    expect(optionsArg.postOptions.params).toEqual({
+      destination_path: '/uploads/',
+      overwrite: 'true'
+    });
   });
 
   it('should call updateFileVariables with correct status on success', async () => {

+ 2 - 1
desktop/core/src/desktop/js/utils/hooks/useFileUpload/useRegularUpload.ts

@@ -55,7 +55,8 @@ const useRegularUpload = ({
       },
       postOptions: {
         params: {
-          destination_path: item.filePath
+          destination_path: item.filePath,
+          overwrite: item.overwrite ? 'true' : 'false'
         },
         onUploadProgress: progress => {
           const itemProgress = getItemProgress(progress);

+ 3 - 2
desktop/core/src/desktop/js/utils/hooks/useFileUpload/utils.ts

@@ -44,7 +44,6 @@ export const getMetaData = (item: ChunkedFile): FileChunkMetaData => ({
 
 export const createChunks = (item: RegularFile, chunkSize: number): ChunkedFile[] => {
   const totalChunks = Math.max(1, getTotalChunk(item.file.size, chunkSize));
-
   const chunks = Array.from({ length: totalChunks }, (_, i) => {
     const chunkStartOffset = i * chunkSize;
     const chunkEndOffset = Math.min(chunkStartOffset + chunkSize, item.file.size);
@@ -56,7 +55,8 @@ export const createChunks = (item: RegularFile, chunkSize: number): ChunkedFile[
       totalChunks,
       chunkNumber: i,
       chunkStartOffset,
-      chunkEndOffset
+      chunkEndOffset,
+      overwrite: item.overwrite
     };
   });
 
@@ -87,6 +87,7 @@ export const getChunkItemPayload = (chunkItem: ChunkedFile): FileUploadApiPayloa
 
   const payload = new FormData();
   payload.append('hdfs_file', chunkItem.file);
+  payload.append('overwrite', chunkItem.overwrite ? 'true' : 'false');
   return { url, payload };
 };