Browse Source

[ui-newimporter] add file preview page (#4088)

Ram Prasad Agarwal 8 months ago
parent
commit
a509c270f0

+ 66 - 0
desktop/core/src/desktop/js/apps/newimporter/ImporterFilePreview/ImporterFilePreview.scss

@@ -0,0 +1,66 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@use 'variables' as vars;
+
+.antd.cuix {
+  .hue-importer-preview-page {
+    display: flex;
+    flex-direction: column;
+    gap: 16px;
+    padding: 8px 16px 16px 24px;
+    height: 100%;
+
+    &__header {
+      display: flex;
+      justify-content: space-between;
+
+      &__title {
+        display: flex;
+        align-items: center;
+        font-size: vars.$fluidx-heading-h3-size;
+        font-weight: vars.$fluidx-heading-h3-weight;
+        color: vars.$fluidx-gray-900;
+      }
+
+      &__actions {
+        display: flex;
+        align-items: center;
+        gap: 24px;
+
+        .action {
+          margin-left: 10px;
+        }
+      }
+    }
+
+    &__metadata {
+      background-color: white;
+      height: 68px;
+      padding: 16px;
+      font-size: 12px;
+      color: #5a656d;
+    }
+
+    &__main-section {
+      display: flex;
+      flex: 1;
+      justify-content: center;
+      background-color: white;
+      padding: 16px 0 0 0;
+    }
+  }
+}

+ 74 - 0
desktop/core/src/desktop/js/apps/newimporter/ImporterFilePreview/ImporterFilePreview.test.tsx

@@ -0,0 +1,74 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import React from 'react';
+import '@testing-library/jest-dom';
+import { render, screen, waitFor } from '@testing-library/react';
+import ImporterFilePreview from './ImporterFilePreview';
+import { FileMetaData } from '../types';
+
+const mockSave = jest.fn();
+jest.mock('../../../utils/hooks/useSaveData/useSaveData', () => ({
+  __esModule: true,
+  default: jest.fn(() => ({
+    data: {
+      columns: [{ name: 'Name' }, { name: 'Age' }],
+      sample: [
+        ['Alice', '30'],
+        ['Bob', '25']
+      ]
+    },
+    save: mockSave,
+    loading: false
+  }))
+}));
+
+describe('ImporterFilePreview', () => {
+  const mockFileMetaData: FileMetaData = {
+    source: 'localfile',
+    type: 'csv',
+    path: '/path/to/file.csv'
+  };
+
+  it('should render correctly', async () => {
+    render(<ImporterFilePreview fileMetaData={mockFileMetaData} />);
+
+    await waitFor(() => {
+      expect(screen.getByText('Preview')).toBeInTheDocument();
+      expect(screen.getByText('Cancel')).toBeInTheDocument();
+      expect(screen.getByText('Finish Import')).toBeInTheDocument();
+    });
+  });
+
+  it('should call guessFormat and guessFields when the component mounts', async () => {
+    render(<ImporterFilePreview fileMetaData={mockFileMetaData} />);
+
+    await waitFor(() => {
+      expect(mockSave).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  it('should display data in the table after previewData is available', async () => {
+    render(<ImporterFilePreview fileMetaData={mockFileMetaData} />);
+
+    await waitFor(() => {
+      expect(screen.getByText('Alice')).toBeInTheDocument();
+      expect(screen.getByText('30')).toBeInTheDocument();
+      expect(screen.getByText('Bob')).toBeInTheDocument();
+      expect(screen.getByText('25')).toBeInTheDocument();
+    });
+  });
+});

+ 121 - 0
desktop/core/src/desktop/js/apps/newimporter/ImporterFilePreview/ImporterFilePreview.tsx

@@ -0,0 +1,121 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import React, { useEffect, useState } from 'react';
+import useSaveData from '../../../utils/hooks/useSaveData/useSaveData';
+import {
+  FileFormatResponse,
+  FileMetaData,
+  GuessFieldTypesResponse,
+  ImporterTableData
+} from '../types';
+import { convertToAntdColumns, convertToDataSource } from '../utils/utils';
+import { i18nReact } from '../../../utils/i18nReact';
+import { BorderlessButton, PrimaryButton } from 'cuix/dist/components/Button';
+import LoadingErrorWrapper from '../../../reactComponents/LoadingErrorWrapper/LoadingErrorWrapper';
+import useResizeObserver from '../../../utils/hooks/useResizeObserver/useResizeObserver';
+import PaginatedTable from '../../../reactComponents/PaginatedTable/PaginatedTable';
+import { GUESS_FORMAT_URL, GUESS_FIELD_TYPES_URL } from '../api';
+
+import './ImporterFilePreview.scss';
+
+interface ImporterFilePreviewProps {
+  fileMetaData: FileMetaData;
+}
+
+const ImporterFilePreview = ({ fileMetaData }: ImporterFilePreviewProps): JSX.Element => {
+  const { t } = i18nReact.useTranslation();
+  const [fileFormat, setFileFormat] = useState<FileFormatResponse | null>(null);
+
+  const { save: guessFormat, loading: guessingFormat } = useSaveData<FileFormatResponse>(
+    GUESS_FORMAT_URL,
+    {
+      onSuccess: data => {
+        setFileFormat(data);
+      }
+    }
+  );
+
+  const {
+    save: guessFields,
+    data: previewData,
+    loading: guessingFields
+  } = useSaveData<GuessFieldTypesResponse>(GUESS_FIELD_TYPES_URL);
+
+  useEffect(() => {
+    const guessFormatPayload = {
+      inputFormat: fileMetaData.source,
+      file_type: fileMetaData.type,
+      path: fileMetaData.path
+    };
+    const guessFormatormData = new FormData();
+    guessFormatormData.append('fileFormat', JSON.stringify(guessFormatPayload));
+    guessFormat(guessFormatormData);
+  }, [fileMetaData]);
+
+  useEffect(() => {
+    if (!fileFormat) {
+      return;
+    }
+
+    const payload = {
+      path: fileMetaData.path,
+      format: fileFormat,
+      inputFormat: fileMetaData.source
+    };
+    const formData = new FormData();
+    formData.append('fileFormat', JSON.stringify(payload));
+    guessFields(formData);
+  }, [fileMetaData.path, fileFormat]);
+
+  const columns = convertToAntdColumns(previewData?.columns ?? []);
+  const tableData = convertToDataSource(columns, previewData?.sample);
+
+  const [ref, rect] = useResizeObserver();
+
+  return (
+    <div className="hue-importer-preview-page">
+      <div className="hue-importer-preview-page__header">
+        <div className="hue-importer-preview-page__header__title">{t('Preview')}</div>
+        <div className="hue-importer-preview-page__header__actions">
+          <BorderlessButton data-testid="hue-importer-preview-page__header__actions__cancel">
+            {t('Cancel')}
+          </BorderlessButton>
+          <PrimaryButton data-testid="hue-importer-preview-page__header__actions__finish">
+            {t('Finish Import')}
+          </PrimaryButton>
+        </div>
+      </div>
+      <div className="hue-importer-preview-page__metadata">{t('DESTINATION')}</div>
+      <div className="hue-importer-preview-page__main-section" ref={ref}>
+        <LoadingErrorWrapper loading={guessingFormat || guessingFields} errors={[]} hideChildren>
+          <PaginatedTable<ImporterTableData>
+            data={tableData}
+            columns={columns}
+            rowKey="importerDataKey"
+            scroll={{
+              y: Math.max(rect.height - 60, 100),
+              x: true
+            }}
+            locale={{ emptyText: t('No data available') }}
+          />
+        </LoadingErrorWrapper>
+      </div>
+    </div>
+  );
+};
+
+export default ImporterFilePreview;

+ 9 - 2
desktop/core/src/desktop/js/apps/newimporter/ImporterPage.tsx

@@ -14,23 +14,30 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import React from 'react';
+import React, { useState } from 'react';
 import DataInIcon from '@cloudera/cuix-core/icons/react/DataInIcon';
 
 import { i18nReact } from '../../utils/i18nReact';
+import { FileMetaData } from './types';
 import CommonHeader from '../../reactComponents/CommonHeader/CommonHeader';
 import ImporterSourceSelector from './ImporterSourceSelector/ImporterSourceSelector';
+import ImporterFilePreview from './ImporterFilePreview/ImporterFilePreview';
 
 import './ImporterPage.scss';
 
 const ImporterPage = (): JSX.Element => {
   const { t } = i18nReact.useTranslation();
+  const [fileMetaData, setFileMetaData] = useState<FileMetaData>();
 
   return (
     <div className="hue-importer cuix antd">
       <CommonHeader title={t('Importer')} icon={<DataInIcon />} />
       <div className="hue-importer__container">
-        <ImporterSourceSelector />
+        {!fileMetaData?.path ? (
+          <ImporterSourceSelector setFileMetaData={setFileMetaData} />
+        ) : (
+          <ImporterFilePreview fileMetaData={fileMetaData} />
+        )}
       </div>
     </div>
   );

+ 43 - 43
desktop/core/src/desktop/js/apps/newimporter/ImporterSourceSelector/ImporterSourceSelector.scss

@@ -20,54 +20,54 @@
 $option-size: 80px;
 
 .antd.cuix {
-    .hue-importer__source-selector{
-        margin: vars.$fluidx-spacing-m;
-        background-color: white;
-        padding: vars.$fluidx-spacing-m 0;
+  .hue-importer__source-selector {
+    margin: vars.$fluidx-spacing-m;
+    background-color: white;
+    padding: vars.$fluidx-spacing-m 0;
 
-        &-title {
-            display: flex;
-            justify-content: center;  
-            align-items: center;
-            font-size: vars.$fluidx-heading-h3-size;
-            line-height: vars.$fluidx-heading-h3-line-height;
-            font-weight: vars.$fluidx-heading-h3-weight;
-        }
+    &-title {
+      display: flex;
+      justify-content: center;
+      align-items: center;
+      font-size: vars.$fluidx-heading-h3-size;
+      line-height: vars.$fluidx-heading-h3-line-height;
+      font-weight: vars.$fluidx-heading-h3-weight;
+    }
 
-        &-options {
-            display: flex;
-            flex-flow: row wrap;
-            justify-content: center; 
-            margin: 0 auto;
-            margin-top: 70px;
-            max-width: 80%;
-            gap: $option-size;     
-        }
+    &-options {
+      display: flex;
+      flex-flow: row wrap;
+      justify-content: center;
+      margin: 0 auto;
+      margin-top: 70px;
+      max-width: 80%;
+      gap: $option-size;
+    }
 
-        &-option {
-            display: flex;
-            flex-direction: column;
-            align-items: center;
-            justify-content: center;
-            width: $option-size;
+    &-option {
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+      justify-content: center;
+      width: $option-size;
 
-            &-button {
-                height: $option-size;
-                width: 100%;
+      &-button {
+        height: $option-size;
+        width: 100%;
 
-                >svg {
-                    height: 40px;
-                }
-            }
+        >svg {
+          height: 40px;
+        }
+      }
 
-            &-btn-title {
-                margin-top: 14px;
-                text-align: center;
-            }
+      &-btn-title {
+        margin-top: 14px;
+        text-align: center;
+      }
 
-            &-upload {
-                display: none;
-            }
-        }
+      &-upload {
+        display: none;
+      }
     }
-}
+  }
+}

+ 14 - 6
desktop/core/src/desktop/js/apps/newimporter/ImporterSourceSelector/ImporterSourceSelector.tsx

@@ -17,8 +17,9 @@
 import React, { useRef } from 'react';
 import Button from 'cuix/dist/components/Button';
 import DocumentationIcon from '@cloudera/cuix-core/icons/react/DocumentationIcon';
-
 import { hueWindow } from 'types/types';
+
+import { FileMetaData, LocalFileUploadResponse } from '../types';
 import { i18nReact } from '../../../utils/i18nReact';
 import { UPLOAD_LOCAL_FILE_API_URL } from '../api';
 import useSaveData from '../../../utils/hooks/useSaveData/useSaveData';
@@ -26,11 +27,15 @@ import huePubSub from '../../../utils/huePubSub';
 
 import './ImporterSourceSelector.scss';
 
-const ImporterSourceSelector = (): JSX.Element => {
+interface ImporterSourceSelectorProps {
+  setFileMetaData: (fileMetaData: FileMetaData) => void;
+}
+
+const ImporterSourceSelector = ({ setFileMetaData }: ImporterSourceSelectorProps): JSX.Element => {
   const { t } = i18nReact.useTranslation();
   const uploadRef = useRef<HTMLInputElement>(null);
 
-  const { save: upload } = useSaveData(UPLOAD_LOCAL_FILE_API_URL);
+  const { save: upload } = useSaveData<LocalFileUploadResponse>(UPLOAD_LOCAL_FILE_API_URL);
 
   const handleUploadClick = () => {
     if (!uploadRef || !uploadRef.current) {
@@ -63,9 +68,12 @@ const ImporterSourceSelector = (): JSX.Element => {
       });
     } else {
       upload(payload, {
-        onSuccess: () => {
-          //TODO: Send response to preview page
-          // console.log(response);
+        onSuccess: data => {
+          setFileMetaData({
+            path: data.local_file_url,
+            type: data.file_type,
+            source: 'localfile'
+          });
         },
         onError: error => {
           huePubSub.publish('hue.error', error);

+ 2 - 0
desktop/core/src/desktop/js/apps/newimporter/api.ts

@@ -15,3 +15,5 @@
 // limitations under the License.
 
 export const UPLOAD_LOCAL_FILE_API_URL = '/indexer/api/indexer/upload_local_file';
+export const GUESS_FORMAT_URL = '/indexer/api/indexer/guess_format';
+export const GUESS_FIELD_TYPES_URL = '/indexer/api/indexer/guess_field_types';

+ 64 - 0
desktop/core/src/desktop/js/apps/newimporter/types.ts

@@ -0,0 +1,64 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export interface LocalFileUploadResponse {
+  local_file_url: string;
+  file_type: string;
+}
+
+export interface FileFormatResponse {
+  fieldSeparator: string;
+  hasHeader: boolean;
+  quoteChar: string;
+  recordSeparator: string;
+  status: number;
+  type: string;
+}
+
+export interface FileMetaData {
+  path: string;
+  type?: string;
+  source: 'localfile' | 'file';
+}
+
+export type GuessFieldTypesColumn = {
+  importerDataKey?: string; // key for identifying unique data row
+  name: string;
+  type?: string;
+  unique?: boolean;
+  keep?: boolean;
+  required?: boolean;
+  multiValued?: boolean;
+  showProperties?: boolean;
+  level?: number;
+  length?: number;
+  keyType?: string;
+  isPartition?: boolean;
+  partitionValue?: string;
+  comment?: string;
+  scale?: number;
+  precision?: number;
+};
+
+export interface GuessFieldTypesResponse {
+  columns: GuessFieldTypesColumn[];
+  sample: string[][];
+}
+
+export interface ImporterTableData {
+  importerDataKey: string;
+  [key: string]: string | number;
+}

+ 60 - 0
desktop/core/src/desktop/js/apps/newimporter/utils/utils.test.ts

@@ -0,0 +1,60 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { ColumnProps } from 'cuix/dist/components/Table';
+import { convertToAntdColumns, convertToDataSource } from './utils';
+import { GuessFieldTypesColumn } from '../types';
+
+describe('convertToAntdColumns', () => {
+  it('should return an empty array when no input is provided', () => {
+    expect(convertToAntdColumns()).toEqual([]);
+  });
+
+  it('should correctly convert GuessFieldTypesColumn[] to ColumnProps[]', () => {
+    const input: GuessFieldTypesColumn[] = [{ name: 'name' }, { name: 'age' }];
+    const expectedOutput = [
+      { title: 'name', dataIndex: 'name', key: 'name', width: '100px' },
+      { title: 'age', dataIndex: 'age', key: 'age', width: '100px' }
+    ];
+
+    expect(convertToAntdColumns(input)).toEqual(expectedOutput);
+  });
+});
+
+describe('convertToDataSource', () => {
+  const columns: ColumnProps<GuessFieldTypesColumn>[] = [
+    { title: 'Name', dataIndex: 'name', key: 'name', width: '100px' },
+    { title: 'Age', dataIndex: 'age', key: 'age', width: '100px' }
+  ];
+
+  it('should return an empty array when no apiResponse is provided', () => {
+    expect(convertToDataSource(columns)).toEqual([]);
+  });
+
+  it('should correctly convert apiResponse to GuessFieldTypesColumn[]', () => {
+    const apiResponse: string[][] = [
+      ['Alice', '30'],
+      ['Bob', '25']
+    ];
+
+    const expectedOutput = [
+      { importerDataKey: 'Alice__0', name: 'Alice', age: '30' },
+      { importerDataKey: 'Bob__1', name: 'Bob', age: '25' }
+    ];
+
+    expect(convertToDataSource(columns, apiResponse)).toEqual(expectedOutput);
+  });
+});

+ 52 - 0
desktop/core/src/desktop/js/apps/newimporter/utils/utils.ts

@@ -0,0 +1,52 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { type ColumnProps } from 'cuix/dist/components/Table';
+import { GuessFieldTypesColumn, ImporterTableData } from '../types';
+
+export const convertToAntdColumns = (
+  input?: GuessFieldTypesColumn[]
+): ColumnProps<ImporterTableData>[] => {
+  if (!input) {
+    return [];
+  }
+  return input?.map(item => ({
+    title: item.name,
+    dataIndex: item.name,
+    key: item.name,
+    width: '100px'
+  }));
+};
+
+export const convertToDataSource = (
+  columns: ColumnProps<ImporterTableData>[],
+  apiResponse?: string[][]
+): ImporterTableData[] => {
+  if (!apiResponse) {
+    return [];
+  }
+  return apiResponse?.map((rowData, index) => {
+    const row = {
+      importerDataKey: `${rowData[0]}__${index}` // this ensure the key is unique
+    };
+    columns.forEach((column, index) => {
+      if (column.key) {
+        row[column.key] = rowData[index];
+      }
+    });
+    return row;
+  });
+};