Explorar o código

[core] Refactor requirements generator along with upgrading ruff and improving its configuration (#4147)

* [ruff] Update ruff version and update its configuration

- Upgrades the linter to a newer version to benefit from improved features and fixes.
- Expands linting rules to include pyflakes, adjusts formatting options for flexibility and consistency, and streamlines import sorting configuration for maintainability.

* [core] Refactor requirements generator for clarity and consistency

 - Improves readability and maintainability by reordering and reformatting requirement lists, aligning dependency order, adding license, and making minor docstring and style updates.

* Add unit tests for requirements generator feature

* Refactor pyproject.toml for improved organization and clarity
Harsh Gupta hai 6 meses
pai
achega
2d547872c8

+ 16 - 0
desktop/core/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 1 - 1
desktop/core/base_requirements.txt

@@ -58,7 +58,7 @@ requests==2.32.3
 requests-kerberos==0.14.0
 krb5==0.5.1  # pinned for Sles12, dep of requests-kerberos 0.14.0
 rsa==4.7.2
-ruff==0.4.2
+ruff==0.11.10
 slack-sdk==3.31.0
 SQLAlchemy==1.3.8
 sqlparse==0.5.0

+ 213 - 194
desktop/core/generate_requirements.py

@@ -1,213 +1,232 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 """
-Dynamic requirements generator for Hue multi-python build system.
+Dynamic requirements generator for the Hue multi-Python build system.
 
-Generates architecture- and Python-version-specific requirement files
-at build time. This tool replaces static requirements.txt and supports:
-  - Local editable paths for Hue extensions.
-  - Binary wheels for specific platforms (e.g., ppc64le).
-  - Architecture-aware dependency lists.
+Generate architecture- and Python-version-specific requirement files at
+build time, replacing a static requirements.txt. This tool supports:
+
+- Local editable paths for Hue extensions.
+- Binary wheels for specific platforms (e.g., ppc64le).
+- Architecture-aware dependency lists.
 """
 
 import os
-import sys
-import shutil
 import platform
+import shutil
+import sys
 
 this_dir = os.path.dirname(os.path.abspath(__file__))
 
 
 class RequirementsGenerator:
-    def __init__(self):
-        self.local_requirements = [
-            "boto-2.49.0",
-            "django-axes-5.13.0",
-            "django-babel",
-            "pysaml2-7.3.1",
-            "python-sasl-0.3.1",
-        ]
+  def __init__(self):
+    self.local_requirements = [
+      "boto-2.49.0",
+      "django-axes-5.13.0",
+      "django-babel",
+      "pysaml2-7.3.1",
+      "python-sasl-0.3.1",
+    ]
+
+    self.requirements = [
+      "setuptools==70.0.0",
+      "apache-ranger==0.0.3",
+      "asn1crypto==0.24.0",
+      "avro-python3==1.8.2",
+      "Babel==2.9.1",
+      "celery[redis]==5.4.0",
+      "cffi==1.15.0",
+      "channels==4.0.0",
+      "channels-redis==4.0.0",
+      "configobj==5.0.9",
+      "cx-Oracle==8.3.0",
+      "daphne==3.0.2",
+      "Django==4.1.13",
+      "django-auth-ldap==4.3.0",
+      "django-celery-beat==2.6.0",
+      "django-celery-results==2.5.1",
+      "django-cors-headers==3.13.0",
+      "django-crequest==2018.5.11",
+      "django-extensions==3.2.1",
+      "django-ipware==3.0.2",
+      "django_opentracing==1.1.0",
+      "django_prometheus==2.3.1",
+      "django-redis==5.4.0",
+      "django-utils-six==2.0",
+      "django-webpack-loader==1.0.0",
+      "djangomako==1.3.2",
+      "djangorestframework==3.14.0",
+      "djangorestframework-simplejwt==5.2.1",
+      "djangosaml2==1.9.3",
+      "drf-spectacular[sidecar]==0.27.2",
+      "future==0.18.3",
+      "gcs-oauth2-boto-plugin==3.0",
+      "greenlet==3.1.1",
+      "gunicorn==23.0.0",
+      "ipython==8.12.2",  # Python >= 3.8
+      "jaeger-client==4.3.0",
+      "jdcal==1.0.1",
+      "kazoo==2.8.0",
+      "kerberos==1.3.0",
+      "krb5==0.5.1",  # pinned for Sles12, dep of requests-kerberos 0.14.0
+      "kubernetes==26.1.0",
+      "Mako==1.2.3",
+      "openpyxl==3.0.9",
+      "phoenixdb==1.2.1",
+      "prompt-toolkit==3.0.39",
+      "protobuf==3.20.3",
+      "psutil==5.8.0",
+      "pyarrow==17.0.0",
+      "pyformance==0.3.2",
+      "PyJWT==2.4.0",
+      "python-daemon==2.2.4",
+      "python-dateutil==2.8.2",
+      "python-ldap==3.4.3",
+      "python-oauth2==1.1.0",
+      "python-pam==2.0.2",
+      "pytidylib==0.3.2",
+      "pytz==2021.3",
+      "PyYAML==6.0.1",
+      "requests==2.32.3",
+      "requests-gssapi==1.2.3",
+      "requests-kerberos==0.14.0",
+      "rsa==4.7.2",
+      "ruff==0.11.10",
+      "six==1.16.0",
+      "slack-sdk==3.31.0",
+      "SQLAlchemy==1.3.8",
+      "sqlparse==0.5.0",
+      "tablib==0.13.0",
+      "tabulate==0.8.9",
+      "thrift-sasl==0.4.3",
+      "trino==0.329.0",
+      "git+https://github.com/gethue/thrift.git",
+    ]
+
+    self.ppc64le_requirements = {
+      "default": [],
+      "3.8": [
+        "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/cryptography-41.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+        "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/lxml-4.6.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+        "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/numpy-1.23.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+        "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/pandas-1.4.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+        "Markdown==3.1",
+        "PyYAML==5.4.1",
+      ],
+      "3.9": [
+        "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/cryptography-41.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+        "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/lxml-4.6.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+        "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/numpy-1.23.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+        "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/pandas-1.4.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+        "Markdown==3.8",
+        "PyYAML==6.0.1",
+      ],
+    }
 
-        self.requirements = [
-            "setuptools==70.0.0",
-            "apache-ranger==0.0.3",
-            "requests-gssapi==1.2.3",
-            "asn1crypto==0.24.0",
-            "avro-python3==1.8.2",
-            "Babel==2.9.1",
-            "celery[redis]==5.4.0",
-            "cffi==1.15.0",
-            "channels==4.0.0",
-            "channels-redis==4.0.0",
-            "configobj==5.0.9",
-            "cx-Oracle==8.3.0",
-            "django-auth-ldap==4.3.0",
-            "Django==4.1.13",
-            "daphne==3.0.2",
-            "django-redis==5.4.0",
-            "django-celery-beat==2.6.0",
-            "django-celery-results==2.5.1",
-            "django-cors-headers==3.13.0",
-            "django-crequest==2018.5.11",
-            "django-extensions==3.2.1",
-            "django-ipware==3.0.2",
-            "django_opentracing==1.1.0",
-            "django_prometheus==2.3.1",
-            "django-webpack-loader==1.0.0",
-            "djangomako==1.3.2",
-            "djangorestframework-simplejwt==5.2.1",
-            "djangorestframework==3.14.0",
-            "djangosaml2==1.9.3",
-            "future==0.18.3",
-            "gcs-oauth2-boto-plugin==3.0",
-            "greenlet==3.1.1",
-            "gunicorn==23.0.0",
-            "ipython==8.12.2",  # Python >= 3.8
-            "jaeger-client==4.3.0",
-            "jdcal==1.0.1",
-            "kazoo==2.8.0",
-            "kerberos==1.3.0",
-            "kubernetes==26.1.0",
-            "Mako==1.2.3",
-            "openpyxl==3.0.9",
-            "phoenixdb==1.2.1",
-            "prompt-toolkit==3.0.39",
-            "protobuf==3.20.3",
-            "pyarrow==17.0.0",
-            "pyformance==0.3.2",
-            "python-dateutil==2.8.2",
-            "python-daemon==2.2.4",
-            "python-ldap==3.4.3",
-            "python-oauth2==1.1.0",
-            "python-pam==2.0.2",
-            "pytidylib==0.3.2",
-            "pytz==2021.3",
-            "PyJWT==2.4.0",
-            "PyYAML==6.0.1",
-            "requests==2.32.3",
-            "requests-kerberos==0.14.0",
-            "krb5==0.5.1",  # pinned for Sles12, dep of requests-kerberos 0.14.0
-            "rsa==4.7.2",
-            "ruff==0.4.2",
-            "slack-sdk==3.31.0",
-            "SQLAlchemy==1.3.8",
-            "sqlparse==0.5.0",
-            "tablib==0.13.0",
-            "tabulate==0.8.9",
-            "trino==0.329.0",
-            "git+https://github.com/gethue/thrift.git",
-            "thrift-sasl==0.4.3",
-            "django-utils-six==2.0",
-            "six==1.16.0",
-            "psutil==5.8.0",
-            "drf-spectacular[sidecar]==0.27.2",
-        ]
-        self.ppc64le_requirements = {
-            "default": [],
-            "3.8": [
-                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/cryptography-41.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
-                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/numpy-1.23.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
-                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/pandas-1.4.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
-                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/lxml-4.6.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
-                "PyYAML==5.4.1",
-                "Markdown==3.1",
-            ],
-            "3.9": [
-                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/cryptography-41.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
-                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/numpy-1.23.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
-                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/pandas-1.4.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
-                "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/lxml-4.6.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
-                "PyYAML==6.0.1",
-                "Markdown==3.8",
-            ]
-        }
-        self.x86_64_requirements = {
-            "default": [
-                "cryptography==42.0.8",
-                "numpy==1.24.4",
-                "pandas==2.0.3",
-                "lxml==4.9.1",
-                "sasl==0.3.1",
-                "Markdown==3.1",
-            ],
-            "3.9": [
-                "pyopenssl==22.1.0",
-                "numpy==1.24.4",
-                "pandas==2.0.3",
-                "lxml==4.9.1",
-                "sasl==0.3.1",
-                "Markdown==3.8",
-                "decorator==5.1.1",
-            ],
-            "3.11": [
-                "cryptography==42.0.8",
-                "numpy==1.24.4",
-                "pandas==2.0.3",
-                "lxml==4.9.1",
-                "async-timeout==5.0.1",
-                "pure-sasl==0.6.2",
-                "Markdown==3.8",
-            ],
-        }
-        self.aarch64_requirements = {
-            "default": [
-                "cryptography==42.0.8",
-                "numpy==1.24.4",
-                "pandas==2.0.3",
-                "lxml==4.9.1",
-                "Markdown==3.1",
-            ],
-            "3.9": [
-                "pyopenssl==22.1.0",
-                "numpy==1.24.4",
-                "pandas==2.0.3",
-                "lxml==4.9.1",
-                "sasl==0.3.1",
-                "Markdown==3.8",
-                "decorator==5.1.1",
-            ],
-            "3.11": [
-                "cryptography==42.0.8",
-                "numpy==1.24.4",
-                "pandas==2.0.3",
-                "lxml==4.9.1",
-                "async-timeout==5.0.1",
-                "pure-sasl==0.6.2",
-                "Markdown==3.8"
-            ],
-        }
-        self.arch_requirements_map = {
-            "ppc64le": self.ppc64le_requirements,
-            "x86_64": self.x86_64_requirements,
-            "aarch64": self.aarch64_requirements,
-        }
-        self.arch = platform.machine()
-        self.python_version_string = f"{sys.version_info.major}.{sys.version_info.minor}"
+    self.x86_64_requirements = {
+      "default": [
+        "cryptography==42.0.8",
+        "lxml==4.9.1",
+        "Markdown==3.1",
+        "numpy==1.24.4",
+        "pandas==2.0.3",
+        "sasl==0.3.1",
+      ],
+      "3.9": [
+        "decorator==5.1.1",
+        "lxml==4.9.1",
+        "Markdown==3.8",
+        "numpy==1.24.4",
+        "pandas==2.0.3",
+        "pyopenssl==22.1.0",
+        "sasl==0.3.1",
+      ],
+      "3.11": [
+        "async-timeout==5.0.1",
+        "cryptography==42.0.8",
+        "lxml==4.9.1",
+        "Markdown==3.8",
+        "numpy==1.24.4",
+        "pandas==2.0.3",
+        "pure-sasl==0.6.2",
+      ],
+    }
 
-    def copy_local_requirements(self, python_version_string):
-        local_dir = f"{this_dir}/{python_version_string}"
-        if os.path.exists(local_dir):
-            shutil.rmtree(local_dir, ignore_errors=True)
-        shutil.copytree(f"{this_dir}/ext-py3", local_dir)
-        return list(map(lambda x: f"file://{local_dir}/{x}", self.local_requirements))
+    self.aarch64_requirements = {
+      "default": [
+        "cryptography==42.0.8",
+        "lxml==4.9.1",
+        "Markdown==3.1",
+        "numpy==1.24.4",
+        "pandas==2.0.3",
+      ],
+      "3.9": [
+        "decorator==5.1.1",
+        "lxml==4.9.1",
+        "Markdown==3.8",
+        "numpy==1.24.4",
+        "pandas==2.0.3",
+        "pyopenssl==22.1.0",
+        "sasl==0.3.1",
+      ],
+      "3.11": [
+        "async-timeout==5.0.1",
+        "cryptography==42.0.8",
+        "lxml==4.9.1",
+        "Markdown==3.8",
+        "numpy==1.24.4",
+        "pandas==2.0.3",
+        "pure-sasl==0.6.2",
+      ],
+    }
+    self.arch_requirements_map = {
+      "ppc64le": self.ppc64le_requirements,
+      "x86_64": self.x86_64_requirements,
+      "aarch64": self.aarch64_requirements,
+    }
+    self.arch = platform.machine()
+    self.python_version_string = f"{sys.version_info.major}.{sys.version_info.minor}"
 
-    def generate_requirements(self):
-        if self.arch not in self.arch_requirements_map:
-            raise ValueError(f"Unsupported architecture: {self.arch}")
-        arch_reqs = self.arch_requirements_map[self.arch]
-        self.requirements.extend(arch_reqs.get(self.python_version_string, arch_reqs["default"]))
-        self.requirements.extend(self.copy_local_requirements(self.python_version_string))
+  def copy_local_requirements(self, python_version_string):
+    local_dir = f"{this_dir}/{python_version_string}"
+    if os.path.exists(local_dir):
+      shutil.rmtree(local_dir, ignore_errors=True)
+    shutil.copytree(f"{this_dir}/ext-py3", local_dir)
+    return list(map(lambda x: f"file://{local_dir}/{x}", self.local_requirements))
 
-        with open(f"{this_dir}/requirements-{self.arch}-{self.python_version_string}.txt", "w") as f:
-            f.write("\n".join(self.requirements))
+  def generate_requirements(self):
+    if self.arch not in self.arch_requirements_map:
+      raise ValueError(f"Unsupported architecture: {self.arch}")
+    arch_reqs = self.arch_requirements_map[self.arch]
+    self.requirements.extend(arch_reqs.get(self.python_version_string, arch_reqs["default"]))
+    self.requirements.extend(self.copy_local_requirements(self.python_version_string))
+    with open(f"{this_dir}/requirements-{self.arch}-{self.python_version_string}.txt", "w") as f:
+      f.write("\n".join(self.requirements))
 
-    def get_file_name(self):
-        return f"{this_dir}/requirements-{self.arch}-{self.python_version_string}.txt"
+  def get_file_name(self):
+    return f"{this_dir}/requirements-{self.arch}-{self.python_version_string}.txt"
 
 
 if __name__ == "__main__":
-    generator = RequirementsGenerator()
-    try:
-        generator.generate_requirements()
-        print(generator.get_file_name())
-    except Exception as e:
-        print(f"Error: {e}")
-        sys.exit(1)
+  generator = RequirementsGenerator()
+  try:
+    generator.generate_requirements()
+    print(generator.get_file_name())
+  except Exception as e:
+    print(f"Error: {e}")
+    sys.exit(1)

+ 235 - 0
desktop/core/generate_requirements_test.py

@@ -0,0 +1,235 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import platform
+import shutil
+import sys
+import tempfile
+from unittest import mock
+
+import pytest
+
+# Workaround for import error: add current directory to Python path
+# The package-style import (desktop.core.generate_requirements) fails,
+# so we modify the path to enable direct module import
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from generate_requirements import RequirementsGenerator
+
+
+class TestRequirementsGenerator:
+  """Test cases for the RequirementsGenerator class."""
+
+  def setup_method(self):
+    """
+    Setup test environment with temporary directory and mock requirement files.
+    Stores original system values to restore after test.
+    """
+    # Store original system values to restore later
+    self.original_arch = platform.machine
+    self.original_version_info = sys.version_info
+    self.temp_dir = tempfile.mkdtemp()
+
+    # Create a temporary ext-py3 directory mimicking the structure used by RequirementsGenerator
+    self.ext_py3_dir = os.path.join(self.temp_dir, "ext-py3")
+    os.makedirs(self.ext_py3_dir)
+
+    # Create empty placeholder files that simulate local requirement packages
+    for req in ["boto-2.49.0", "django-axes-5.13.0", "django-babel", "pysaml2-7.3.1", "python-sasl-0.3.1"]:
+      with open(os.path.join(self.ext_py3_dir, req), "w") as f:
+        f.write("")
+
+  def teardown_method(self):
+    """Teardown after each test method."""
+    shutil.rmtree(self.temp_dir)
+    platform.machine = self.original_arch
+    sys.version_info = self.original_version_info
+
+  @mock.patch("generate_requirements.this_dir")
+  @mock.patch("generate_requirements.platform.machine")
+  @mock.patch("generate_requirements.sys.version_info")
+  def test_init_sets_properties(self, mock_version_info, mock_machine, mock_this_dir):
+    """Test that __init__ sets the class properties correctly."""
+
+    mock_machine.return_value = "x86_64"
+    mock_version_info.major = 3
+    mock_version_info.minor = 9
+    mock_this_dir.return_value = self.temp_dir
+
+    generator = RequirementsGenerator()
+
+    assert generator.arch == "x86_64"
+    assert generator.python_version_string == "3.9"
+    assert isinstance(generator.requirements, list)
+    assert isinstance(generator.local_requirements, list)
+    assert "x86_64" in generator.arch_requirements_map
+    assert "ppc64le" in generator.arch_requirements_map
+    assert "aarch64" in generator.arch_requirements_map
+
+  @mock.patch("generate_requirements.this_dir", create=True)
+  @mock.patch("generate_requirements.shutil.copytree")
+  def test_copy_local_requirements(self, mock_copytree, mock_this_dir):
+    """Test that copy_local_requirements copies files and returns correct paths."""
+
+    # Need to make this_dir return the string value directly, not a MagicMock
+    mock_this_dir.__str__.return_value = self.temp_dir
+    python_version_string = "3.9"
+
+    with mock.patch.object(RequirementsGenerator, "__init__", return_value=None):
+      generator = RequirementsGenerator()
+      generator.local_requirements = ["boto-2.49.0", "django-axes-5.13.0"]
+
+      result = generator.copy_local_requirements(python_version_string)
+
+      mock_copytree.assert_called_once_with(f"{self.temp_dir}/ext-py3", f"{self.temp_dir}/{python_version_string}")
+
+      assert len(result) == 2
+      assert f"file://{self.temp_dir}/{python_version_string}/boto-2.49.0" in result
+      assert f"file://{self.temp_dir}/{python_version_string}/django-axes-5.13.0" in result
+
+  @mock.patch("generate_requirements.this_dir")
+  @mock.patch("builtins.open", new_callable=mock.mock_open)
+  def test_generate_requirements_x86_64(self, mock_open, mock_this_dir):
+    """Test generating requirements for x86_64 architecture."""
+
+    # Need to make this_dir return the string value directly, not a MagicMock
+    mock_this_dir.__str__.return_value = self.temp_dir
+
+    with mock.patch.object(RequirementsGenerator, "__init__", return_value=None):
+      generator = RequirementsGenerator()
+      generator.arch = "x86_64"
+      generator.python_version_string = "3.9"
+      generator.requirements = ["setuptools==70.0.0", "Django==4.1.13"]
+      generator.local_requirements = []
+      generator.arch_requirements_map = {"x86_64": {"default": ["cryptography==42.0.8"], "3.9": ["Markdown==3.8", "numpy==1.24.4"]}}
+
+      # Mock copy_local_requirements for test isolation
+      generator.copy_local_requirements = mock.MagicMock(return_value=[])
+
+      generator.generate_requirements()
+
+      # Verify file creation with correct path
+      mock_open.assert_called_once_with(f"{self.temp_dir}/requirements-x86_64-3.9.txt", "w")
+
+      # Verify correct requirements were written
+      expected_requirements = "\n".join(["setuptools==70.0.0", "Django==4.1.13", "Markdown==3.8", "numpy==1.24.4"])
+      mock_open().write.assert_called_once_with(expected_requirements)
+
+  @mock.patch("generate_requirements.this_dir")
+  def test_generate_requirements_unsupported_arch(self, mock_this_dir):
+    """Test that generate_requirements raises an error for unsupported architectures."""
+
+    # Need to make this_dir return the string value directly, not a MagicMock
+    mock_this_dir.__str__.return_value = self.temp_dir
+
+    with mock.patch.object(RequirementsGenerator, "__init__", return_value=None):
+      generator = RequirementsGenerator()
+      generator.arch = "unsupported_arch"
+      generator.python_version_string = "3.9"
+      generator.requirements = []
+      generator.arch_requirements_map = {"x86_64": {"default": []}, "aarch64": {"default": []}}
+
+      with pytest.raises(ValueError) as exc_info:
+        generator.generate_requirements()
+
+      assert "Unsupported architecture: unsupported_arch" in str(exc_info.value)
+
+  @mock.patch("generate_requirements.this_dir")
+  def test_get_file_name(self, mock_this_dir):
+    """Test that get_file_name returns the correct file path."""
+
+    # Need to make this_dir return the string value directly, not a MagicMock
+    mock_this_dir.__str__.return_value = self.temp_dir
+
+    with mock.patch.object(RequirementsGenerator, "__init__", return_value=None):
+      generator = RequirementsGenerator()
+      generator.arch = "x86_64"
+      generator.python_version_string = "3.9"
+
+      result = generator.get_file_name()
+
+      assert result == f"{self.temp_dir}/requirements-x86_64-3.9.txt"
+
+  @mock.patch("generate_requirements.this_dir")
+  @mock.patch("builtins.open", new_callable=mock.mock_open)
+  def test_generate_requirements_ppc64le(self, mock_open, mock_this_dir):
+    """Test generating requirements for ppc64le architecture."""
+
+    # Need to make this_dir return the string value directly, not a MagicMock
+    mock_this_dir.__str__.return_value = self.temp_dir
+
+    with mock.patch.object(RequirementsGenerator, "__init__", return_value=None):
+      generator = RequirementsGenerator()
+      generator.arch = "ppc64le"
+      generator.python_version_string = "3.8"
+      generator.requirements = ["setuptools==70.0.0"]
+      generator.local_requirements = []
+      generator.arch_requirements_map = {
+        "ppc64le": {
+          "default": [],
+          "3.8": [
+            "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/numpy-1.23.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+            "Markdown==3.1",
+          ],
+        }
+      }
+
+      generator.copy_local_requirements = mock.MagicMock(return_value=[])
+
+      generator.generate_requirements()
+
+      mock_open.assert_called_once_with(f"{self.temp_dir}/requirements-ppc64le-3.8.txt", "w")
+
+      expected_requirements = "\n".join(
+        [
+          "setuptools==70.0.0",
+          "http://ibm-ppc-builds.s3.amazonaws.com/silx-py-libs/numpy-1.23.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl",
+          "Markdown==3.1",
+        ]
+      )
+
+      mock_open().write.assert_called_once_with(expected_requirements)
+
+  @mock.patch("generate_requirements.this_dir")
+  @mock.patch("builtins.open", new_callable=mock.mock_open)
+  def test_generate_requirements_with_local_requirements(self, mock_open, mock_this_dir):
+    """Test generating requirements including local requirements."""
+
+    # Need to make this_dir return the string value directly, not a MagicMock
+    mock_this_dir.__str__.return_value = self.temp_dir
+
+    with mock.patch.object(RequirementsGenerator, "__init__", return_value=None):
+      generator = RequirementsGenerator()
+      generator.arch = "x86_64"
+      generator.python_version_string = "3.9"
+      generator.requirements = ["setuptools==70.0.0"]
+      generator.local_requirements = ["boto-2.49.0", "django-axes-5.13.0"]
+      generator.arch_requirements_map = {"x86_64": {"default": [], "3.9": ["Markdown==3.8"]}}
+
+      # Mock copy_local_requirements to return file paths
+      local_reqs = [f"file://{self.temp_dir}/3.9/boto-2.49.0", f"file://{self.temp_dir}/3.9/django-axes-5.13.0"]
+      generator.copy_local_requirements = mock.MagicMock(return_value=local_reqs)
+
+      generator.generate_requirements()
+
+      mock_open.assert_called_once_with(f"{self.temp_dir}/requirements-x86_64-3.9.txt", "w")
+
+      expected_requirements = "\n".join(
+        ["setuptools==70.0.0", "Markdown==3.8", f"file://{self.temp_dir}/3.9/boto-2.49.0", f"file://{self.temp_dir}/3.9/django-axes-5.13.0"]
+      )
+
+      mock_open().write.assert_called_once_with(expected_requirements)

+ 133 - 119
pyproject.toml

@@ -1,3 +1,5 @@
+# pyproject.toml for Hue
+
 [tool.pytest.ini_options]
 DJANGO_SETTINGS_MODULE = "desktop.settings"
 console_output_style = "count"
@@ -17,69 +19,69 @@ addopts = """
 norecursedirs = "desktop/core/ext-py3 desktop/core/3.11 desktop/core/3.10 desktop/core/3.9 desktop/core/3.8"
 python_files = "tests.py test_*.py *_tests.py tests_* *_test.py"
 markers = [
-    "integration: live server based tests",
-    "requires_hadoop: live hadoop cluster based tests",
+  "integration: live server based tests",
+  "requires_hadoop: live hadoop cluster based tests"
 ]
 
 [tool.coverage.run]
 branch = true
 omit = [
-    "*/ext-py3/*",
-    "desktop/core/src/desktop/lib/wsgiserver.py",
-    "*/migrations/*",
-    "*/org_migrations/*",
-    "*/old_migrations/*",
-    "*/gen-py/*",
-    "desktop/core/src/desktop/app_template/src/app_name/*",
-    "desktop/core/src/desktop/app_template_proxy/src/app_name/*",
-    "*conf.py",
-    "*settings.py",
-    "*urls.py",
-    "*metrics.py",
-    "*tests.py*",
-    "test_*.py",
-    "*_tests.py",
-    "tests_*.py",
-    "*_test.py",
+  "*/ext-py3/*",
+  "desktop/core/src/desktop/lib/wsgiserver.py",
+  "*/migrations/*",
+  "*/org_migrations/*",
+  "*/old_migrations/*",
+  "*/gen-py/*",
+  "desktop/core/src/desktop/app_template/src/app_name/*",
+  "desktop/core/src/desktop/app_template_proxy/src/app_name/*",
+  "*conf.py",
+  "*settings.py",
+  "*urls.py",
+  "*metrics.py",
+  "*tests.py*",
+  "test_*.py",
+  "*_tests.py",
+  "tests_*.py",
+  "*_test.py"
 ]
 source = [
-    "about",
-    "beeswax",
-    "filebrowser",
-    "hbase",
-    "help",
-    "hive",
-    "impala",
-    "jobbrowser",
-    "jobsub",
-    "metastore",
-    "oozie",
-    "pig",
-    "proxy",
-    "rdbms",
-    "search",
-    "security",
-    "spark",
-    "sqoop",
-    "useradmin",
-    "zookeeper",
-    "desktop",
-    "aws",
-    "azure",
-    "dashboard",
-    "hadoop",
-    "indexer",
-    "kafka",
-    "libanalyze",
-    "liboauth",
-    "liboozie",
-    "librdbms",
-    "libsaml",
-    "libsentry",
-    "libsolr",
-    "libzookeeper",
-    "metadata",
-    "notebook",
+  "about",
+  "aws",
+  "azure",
+  "beeswax",
+  "dashboard",
+  "desktop",
+  "filebrowser",
+  "hadoop",
+  "hbase",
+  "help",
+  "hive",
+  "impala",
+  "indexer",
+  "jobbrowser",
+  "jobsub",
+  "kafka",
+  "libanalyze",
+  "liboauth",
+  "liboozie",
+  "librdbms",
+  "libsaml",
+  "libsentry",
+  "libsolr",
+  "libzookeeper",
+  "metadata",
+  "metastore",
+  "notebook",
+  "oozie",
+  "pig",
+  "proxy",
+  "rdbms",
+  "search",
+  "security",
+  "spark",
+  "sqoop",
+  "useradmin",
+  "zookeeper"
 ]
 
 [tool.coverage.report]
@@ -91,28 +93,89 @@ line-length = 140
 indent-width = 2
 force-exclude = true
 extend-exclude = [
-    "*/ext-py3/*",
-    "desktop/core/src/desktop/lib/wsgiserver.py",
-    "*/migrations/*",
-    "apps/oozie/src/oozie/tests.py",
-    "tools/ops/",
-    "tools/ace-editor/",
-    "*/gen-py/*",
-    "*/org_migrations/*",
-    "*/old_migrations/*",
-    "desktop/core/src/desktop/app_template/src/app_name/*",
-    "desktop/core/src/desktop/app_template_proxy/src/app_name/*",
+  "*/ext-py3/*",
+  "desktop/core/src/desktop/lib/wsgiserver.py",
+  "*/migrations/*",
+  "apps/oozie/src/oozie/tests.py",
+  "tools/ops/",
+  "tools/ace-editor/",
+  "*/gen-py/*",
+  "*/org_migrations/*",
+  "*/old_migrations/*",
+  "desktop/core/src/desktop/app_template/src/app_name/*",
+  "desktop/core/src/desktop/app_template_proxy/src/app_name/*"
 ]
 
+[tool.ruff.format]
+docstring-code-format = true
+docstring-code-line-length = "dynamic"
+indent-style = "space"
+quote-style = "double"
+
 [tool.ruff.lint]
 preview = true
 select = [
-    "E", # pycodestyle errors
-    "W", # pycodestyle warnings
-    "I", # isort
+  "E", # pycodestyle errors
+  "W", # pycodestyle warnings
+  "I", # isort
+  "F" # pyflakes
 ]
 ignore = ["E111", "E114", "E117", "W191", "E731"]
 
+[tool.ruff.lint.isort]
+case-sensitive = false
+combine-as-imports = true
+force-sort-within-sections = false
+known-first-party = [
+  "about",
+  "aws",
+  "azure",
+  "beeswax",
+  "dashboard",
+  "desktop",
+  "filebrowser",
+  "hadoop",
+  "hbase",
+  "help",
+  "hive",
+  "impala",
+  "indexer",
+  "jobbrowser",
+  "jobsub",
+  "kafka",
+  "libanalyze",
+  "liboauth",
+  "liboozie",
+  "librdbms",
+  "libsaml",
+  "libsentry",
+  "libsolr",
+  "libzookeeper",
+  "metadata",
+  "metastore",
+  "notebook",
+  "oozie",
+  "pig",
+  "proxy",
+  "rdbms",
+  "search",
+  "security",
+  "spark",
+  "sqoop",
+  "useradmin",
+  "zookeeper"
+]
+known-third-party = []
+lines-after-imports = -1
+order-by-type = false
+section-order = [
+  "future",
+  "standard-library",
+  "third-party",
+  "first-party",
+  "local-folder"
+]
+
 [tool.ruff.lint.per-file-ignores]
 "desktop/libs/notebook/src/notebook/connectors/altus.py" = ["E501"]
 "desktop/libs/notebook/src/notebook/connectors/altus_adb.py" = ["E501"]
@@ -124,52 +187,3 @@ ignore = ["E111", "E114", "E117", "W191", "E731"]
 "desktop/core/src/desktop/urls.py" = ["E402"]
 "desktop/core/src/desktop/lib/thrift_util_test.py" = ["E402"]
 "desktop/core/src/desktop/auth/backend.py" = ["E402"]
-
-[tool.ruff.format]
-docstring-code-format = true
-docstring-code-line-length = 140
-indent-style = "space"
-quote-style = "preserve"
-
-[tool.ruff.lint.isort]
-combine-as-imports = true
-length-sort-straight = true
-known-first-party = [
-    "about",
-    "beeswax",
-    "filebrowser",
-    "hbase",
-    "help",
-    "hive",
-    "impala",
-    "jobbrowser",
-    "jobsub",
-    "metastore",
-    "oozie",
-    "pig",
-    "proxy",
-    "rdbms",
-    "search",
-    "security",
-    "spark",
-    "sqoop",
-    "useradmin",
-    "zookeeper",
-    "desktop",
-    "aws",
-    "azure",
-    "dashboard",
-    "hadoop",
-    "indexer",
-    "kafka",
-    "libanalyze",
-    "liboauth",
-    "liboozie",
-    "librdbms",
-    "libsaml",
-    "libsentry",
-    "libsolr",
-    "libzookeeper",
-    "metadata",
-    "notebook",
-]