commitflow-py3.yml 4.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116
  1. name: Python 3 CI
  2. on:
  3. push:
  4. branches:
  5. - master
  6. pull_request:
  7. branches:
  8. - master
  9. jobs:
  10. build:
  11. runs-on: ${{ (matrix.python-version == '3.10' || matrix.python-version == '3.11') && 'ubuntu-22.04' || 'ubuntu-20.04' }}
  12. strategy:
  13. matrix:
  14. python-version: ['3.8', '3.9', '3.10', '3.11']
  15. steps:
  16. - name: Checkout
  17. uses: actions/checkout@v4
  18. with:
  19. fetch-depth: 0
  20. - name: run commit title format check
  21. run: |
  22. ./tools/ci/check_for_commit_message.sh
  23. - name: Set up Python ${{ matrix.python-version }}
  24. uses: actions/setup-python@v5
  25. with:
  26. python-version: ${{ matrix.python-version }}
  27. - name: Cache pip
  28. uses: actions/cache@v4
  29. with:
  30. # This path is specific to Ubuntu
  31. path: ~/.cache/pip
  32. key: ${{ runner.os }}-pip-${{ hashFiles('desktop/core/requirements.txt') }}
  33. restore-keys: |
  34. ${{ runner.os }}-pip-
  35. ${{ runner.os }}-
  36. - name: Setup node 20 and cache npm
  37. uses: actions/setup-node@v4
  38. with:
  39. node-version: 20
  40. cache: 'npm'
  41. - name: compile
  42. run: |
  43. sudo apt-get update
  44. sudo apt-get install -y gcc g++ build-essential python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils asciidoc rsync curl sudo libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev # This should not be needed as some point
  45. sudo curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python${{ matrix.python-version }}
  46. sudo apt-get install -y python3-setuptools
  47. sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
  48. export PYTHON_VER=python${{ matrix.python-version }}
  49. export ROOT=$PWD
  50. make test_prep
  51. - name: run tests
  52. run: |
  53. mkdir -p test-reports # Ensure the directory exists
  54. mkdir -p code-coverage-reports # Ensure the directory exists
  55. ./build/env/bin/pytest --cov=about --cov=beeswax --cov=filebrowser --cov=hbase --cov=help --cov=hive --cov=impala --cov=jobbrowser --cov=jobsub --cov=metastore --cov=oozie --cov=pig --cov=proxy --cov=rdbms --cov=search --cov=security --cov=spark --cov=sqoop --cov=useradmin --cov=zookeeper --cov=desktop --cov=aws --cov=azure --cov=dashboard --cov=hadoop --cov=indexer --cov=kafka --cov=libanalyze --cov=liboauth --cov=liboozie --cov=librdbms --cov=libsaml --cov=libsentry --cov=libsolr --cov=libzookeeper --cov=metadata --cov=notebook --cov-report=html:code-coverage-report --html=test-reports/report_${{ matrix.python-version }}.html --self-contained-html
  56. - name: upload pytest report
  57. uses: actions/upload-artifact@v4
  58. with:
  59. name: test-reports-${{ matrix.python-version }}
  60. path: test-reports
  61. - name: upload code coverage report
  62. uses: actions/upload-artifact@v4
  63. with:
  64. name: code-coverage-report-${{ matrix.python-version }}
  65. path: code-coverage-report
  66. - name: Check and comment if no unit test files are modified
  67. if: matrix.python-version == '3.11'
  68. run: |
  69. git fetch origin master
  70. changed_files=$(git diff --name-only origin/master)
  71. if echo "$changed_files" | grep -qE '(^test|_test\.py|^tests|_tests\.py|.test)'; then
  72. echo "✅ Unit test files were modified."
  73. else
  74. echo "⚠️ No unit test files modified."
  75. curl -X POST \
  76. -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
  77. -H "Accept: application/vnd.github.v3+json" \
  78. -d '{"body":"⚠️ No unit test files modified. Please ensure that changes are properly tested. ⚠️"}' \
  79. "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments"
  80. fi
  81. - name: run python lints
  82. run: |
  83. PYTHONWARNINGS=always ./build/env/bin/hue runruff check
  84. - name: run documentation lints
  85. run: |
  86. curl -O https://dl.google.com/go/go1.23.4.linux-amd64.tar.gz
  87. tar -xvf go1.23.4.linux-amd64.tar.gz
  88. go/bin/go install github.com/raviqqe/muffet/v2@latest
  89. curl --output hugo_0.69.0_Linux.tar.gz -L https://github.com/gohugoio/hugo/releases/download/v0.69.0/hugo_0.69.0_Linux-64bit.tar.gz
  90. tar -xvf hugo_0.69.0_Linux.tar.gz
  91. export PATH=$PWD:$HOME/go/bin:$PATH
  92. # Trigger linting if documentation changes
  93. ./tools/ci/check_for_website_dead_links.sh docs/docs-site
  94. # ./tools/ci/check_for_website_dead_links.sh docs/gethue