瀏覽代碼

[ruff] Add custom Ruff command in Hue and improve config file (#3716)

## What changes were proposed in this pull request?

- Added a custom Django command in Hue for running ruff. 
This will help in not depending on tools/ci/check_for_python_lint.sh file and have all ruff functionality on the go via Hue command (both linting + formatting).
- Added an extra argument to specify the diff branch to compare the changed files. This will help in scenarios where the base development branch is not always `origin/master`.

- Improved the ruff configs to support formatting of python files and also updated Ruff to latest version.

## How was this patch tested?

- Manually in local setup.
Harsh Gupta 1 年之前
父節點
當前提交
0b8bace2ac
共有 3 個文件被更改,包括 150 次插入2 次删除
  1. 1 1
      desktop/core/base_requirements.txt
  2. 142 0
      desktop/core/src/desktop/management/commands/runruff.py
  3. 7 1
      pyproject.toml

+ 1 - 1
desktop/core/base_requirements.txt

@@ -59,7 +59,7 @@ PyJWT==2.4.0
 PyYAML==6.0.1
 requests-kerberos==0.14.0
 rsa==4.7.2
-ruff==0.3.7
+ruff==0.4.2
 sasl==0.3.1  # Move to https://pypi.org/project/sasl3/ ?
 slack-sdk==3.2.0
 SQLAlchemy==1.3.8

+ 142 - 0
desktop/core/src/desktop/management/commands/runruff.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os.path
+import subprocess
+import sys
+import argparse
+
+from django.core.management.base import BaseCommand
+from django.utils.translation import gettext as _
+
+from desktop.lib import paths
+
+
+LOG = logging.getLogger()
+
+
+class Command(BaseCommand):
+  help = _("""
+    Run Ruff code quality checks on Python files.
+
+    This command runs Ruff on a set of Python files, either specified explicitly or
+    determined by comparing the current branch with a specified diff branch.
+
+    Options:
+      --diff-branch=<branch>  Compare the current branch with the specified branch
+                              and check only the files that have changed.
+      All original Ruff commands are supported, including --fix to automatically fix errors.
+
+    Examples:
+      Run Ruff linting on specific Python files:
+        ./build/env/bin/hue runruff check file1.py file2.py
+
+      Run Ruff linting on files changed between the current branch and a specified diff branch:
+        ./build/env/bin/hue runruff check --diff-branch=origin/master
+
+      Automatically fix all Ruff errors on files changed between the current branch and a specified diff branch:
+        ./build/env/bin/hue runruff check --diff-branch=origin/master --fix
+
+    Note:
+      Make sure to install Ruff first by running `./build/env/bin/pip install ruff` in your environment.
+      This command passes all additional arguments to Ruff, so you can use any Ruff option or flag.
+  """)
+
+  def __init__(self):
+    super().__init__()
+
+  def diff_files(self, diff_branch):
+    """
+    Returns a list of files that have changed between the current branch and the specified diff_branch.
+
+    Args:
+      diff_branch (str): The branch to compare with.
+
+    Returns:
+      list: A list of files that have changed.
+    """
+    git_cmd = ["git", "diff", "--name-only", diff_branch, "--diff-filter=bd"]
+    egrep_cmd = ["egrep", ".py$"]
+
+    try:
+      # Run the git command first and capture it's output
+      git_process = subprocess.run(git_cmd, stdout=subprocess.PIPE, check=True)
+      git_output, _ = git_process.stdout, git_process.stderr
+
+      # Next, run the egrep command on git output and capture final output to return as list of files
+      egrep_process = subprocess.run(egrep_cmd, input=git_output, stdout=subprocess.PIPE, check=True)
+      egrep_output, _ = egrep_process.stdout, egrep_process.stderr
+
+      # Decode the egrep output, strip trailing newline, split into a list of files, and return it
+      # Otherwise, return an empty list
+      return egrep_output.decode().strip('\n').split('\n') if egrep_output else []
+
+    except subprocess.CalledProcessError as e:
+      LOG.error(f"Error running git or egrep command: {e}")
+      return []
+
+  def add_arguments(self, parser):
+    parser.add_argument('ruff_args', nargs=argparse.REMAINDER, help='Additional Ruff arguments, e.g. check, format, --fix etc.')
+    parser.add_argument(
+      '--diff-branch',
+      action='store',
+      dest='diff_branch',
+      default='origin/master',
+      help='Compare with this branch to check only changed files',
+    )
+
+  def handle(self, *args, **options):
+    """
+    Handles the command.
+
+    Args:
+      *args: Variable arguments.
+      **options: Keyword arguments.
+    """
+    ruff_package = paths.get_build_dir('env', 'bin', 'ruff')
+
+    if not os.path.exists(ruff_package):
+      msg = _(
+        "Ruff is not installed. Please run `./build/env/bin/pip install ruff` and try again. Make sure you are in the correct virtualenv."
+      )
+      self.stderr.write(msg + '\n')
+      sys.exit(1)
+
+    ruff_cmd = [ruff_package] + options.get('ruff_args')
+
+    # Exit with a status code of 0 even if linting violations were found and only fail for actual error
+    # Skip adding when running ruff format command
+    if not any(arg in ruff_cmd for arg in ["--exit-zero", "format"]):
+      ruff_cmd.append("--exit-zero")
+
+    if options.get('diff_branch'):
+      diff_files = self.diff_files(options['diff_branch'])
+
+      if not diff_files:
+        self.stdout.write(_("No Python code files changes present.") + '\n')
+        return None
+
+      ruff_cmd += diff_files
+
+    try:
+      ret = subprocess.run(ruff_cmd, check=True)
+      if ret.returncode != 0:
+        sys.exit(1)
+    except subprocess.CalledProcessError as e:
+      LOG.error(f"Error running ruff command: {e}")
+      sys.exit(1)

+ 7 - 1
pyproject.toml

@@ -11,6 +11,7 @@ markers = [
 [tool.ruff]
 target-version = "py38"
 line-length = 140
+indent-width = 2
 force-exclude = true
 extend-exclude = [
   "*/ext-py3/*",
@@ -20,7 +21,8 @@ extend-exclude = [
   "tools/ops/",
   "tools/ace-editor/",
   "*/gen-py/*",
-  "*/org_migrations/*"
+  "*/org_migrations/*",
+  "*/old_migrations/*"
 ]
 
 [tool.ruff.lint]
@@ -32,8 +34,12 @@ select = [
 ignore = [
     "E111",
     "E114",
+    "E117",
     "W191",
 ]
 
 [tool.ruff.format]
 docstring-code-format = true
+docstring-code-line-length = 140
+indent-style = "space"
+quote-style = "preserve"