Browse Source

Streamlines CI workflows and standardizes test environments

- Standardizes CI runners to ubuntu-latest for consistency
- Updates dependencies and build requirements to minimal needed set
- Improves test execution with better caching and error handling
- Enhances lint checks and reporting across Python and JS workflows
- Renames jobs for better clarity in GitHub Actions UI

Note: Reduces system dependencies and build time while maintaining full test coverage
Harsh Gupta 7 months ago
parent
commit
5272c5b31b

+ 10 - 10
.github/workflows/commitflow-frontend.yml

@@ -9,13 +9,13 @@ on:
     - master
 
 jobs:
-  build:
-    runs-on: ubuntu-20.04
+  hue-ui-checks-and-tests:
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v4
 
-    - name: Caching npm with setup node
+    - name: Cache npm with setup node
       uses: actions/setup-node@v4
       with:
         node-version: 20
@@ -24,25 +24,25 @@ jobs:
     - name: Install npm dependencies
       run: npm ci
 
-    - name: run commit title format check
+    - name: Run commit title format check
       run: ./tools/ci/check_for_commit_message.sh 
 
-    - name: run npm version checker
+    - name: Run npm version checker
       run: npm run check-pinned-versions
 
-    - name: run npm license checker
+    - name: Run npm license checker
       run: npm run check-license
 
-    - name: run npm absolute path detection
+    - name: Run npm absolute path detection
       run: npm run check-absolute-paths
 
-    - name: run js lint
+    - name: Run js lint check
       run: npm run lint
 
-    - name: run style lint
+    - name: Run style lint check
       run: npm run style-lint
 
-    - name: run js tests
+    - name: Run js tests
       run: |
         # https://jestjs.io/docs/en/troubleshooting.html#tests-are-extremely-slow-on-docker-andor-continuous-integration-ci-server
         sed -i 's/"test": "jest"/"test": "jest --runInBand"/g' package.json

+ 8 - 10
.github/workflows/commitflow-py3.yml

@@ -8,9 +8,9 @@ on:
     branches:
     - master
 jobs:
-  build:
+  hue-build-and-tests:
 
-    runs-on: ${{ (matrix.python-version == '3.10' || matrix.python-version == '3.11') && 'ubuntu-22.04' || 'ubuntu-20.04' }}
+    runs-on: ubuntu-latest
 
     strategy:
       matrix:
@@ -36,7 +36,7 @@ jobs:
       with:
         # This path is specific to Ubuntu
         path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('desktop/core/requirements.txt') }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('desktop/core/generate_requirements.py') }}
         restore-keys: |
           ${{ runner.os }}-pip-
           ${{ runner.os }}-
@@ -50,16 +50,14 @@ jobs:
     - name: Build Hue
       run: |
         sudo apt-get update
-        sudo apt-get install -y gcc g++ build-essential python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils asciidoc rsync curl sudo libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev  libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev # This should not be needed as some point
-        sudo curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python${{ matrix.python-version }}
-        sudo apt-get install -y python3-setuptools
-        sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
+        sudo apt-get install -y build-essential asciidoc libkrb5-dev libldap2-dev libsasl2-dev libxslt-dev libsasl2-modules-gssapi-mit libsnappy-dev libgdbm-dev
 
         export PYTHON_VER=python${{ matrix.python-version }}
+        export SKIP_PYTHONDEV_CHECK=true
         export ROOT=$PWD
         make apps test_prep
 
-    - name: Run python unit tests
+    - name: Run unit tests
       run: |
         if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
           ./build/env/bin/pytest
@@ -67,7 +65,7 @@ jobs:
           ./build/venvs/python${{ matrix.python-version }}/bin/pytest
         fi
 
-    - name: Run python lints
+    - name: Run python lint check
       run: |
         if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
           PYTHONWARNINGS=always ./build/env/bin/hue runruff check
@@ -75,7 +73,7 @@ jobs:
           PYTHONWARNINGS=always ./build/venvs/python${{ matrix.python-version }}/bin/hue runruff check
         fi
 
-    - name: Run documentation lints
+    - name: Run documentation lint check
       run: |
         curl -O https://dl.google.com/go/go1.23.4.linux-amd64.tar.gz
         tar -xvf go1.23.4.linux-amd64.tar.gz

+ 10 - 10
.github/workflows/pr-comments.yml

@@ -10,7 +10,8 @@ permissions:
 
 jobs:
   pytest-codecov-comment:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -25,7 +26,7 @@ jobs:
         with:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('desktop/core/requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('desktop/core/generate_requirements.py') }}
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
@@ -39,16 +40,15 @@ jobs:
       - name: Build Hue
         run: |
           sudo apt-get update
-          sudo apt-get install -y gcc g++ build-essential python3.11-dev python3.11-venv python3.11-distutils asciidoc rsync curl sudo libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev  libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev # This should not be needed as some point
-          sudo curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python3.11
-          sudo apt-get install -y python3-setuptools
-          sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
+          sudo apt-get install -y build-essential asciidoc libkrb5-dev libldap2-dev libsasl2-dev libxslt-dev libsasl2-modules-gssapi-mit libsnappy-dev libgdbm-dev
 
           export PYTHON_VER=python3.11
+          export SKIP_PYTHONDEV_CHECK=true
           export ROOT=$PWD
           make apps test_prep
 
       - name: Run python unit tests
+        continue-on-error: true
         run: |
           ./build/env/bin/pytest
       
@@ -58,15 +58,15 @@ jobs:
           pytest-xml-coverage-path: ./reports/code-cov/coverage.xml
           junitxml-path: ./reports/pytest/test_report.xml
           junitxml-title: Pytest Report
-          title: Backend Code Coverage Report
-          badge-title: Backend Codecov
+          title: Python Coverage Report
+          badge-title: Python Code Coverage
           report-only-changed-files: true
           xml-skip-covered: true
           remove-link-from-badge: true
           default-branch: master
   
   python-ut-files-comment:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -86,7 +86,7 @@ jobs:
         id: check-test-files
         run: |
           if [[ "${{ steps.changed-test-files.outputs.test_any_changed }}" == "true" ]]; then
-            echo "comment_message=✅ Test files were modified. Ensure that the tests cover all relevant changes" >> $GITHUB_ENV
+            echo "comment_message=✅ Test files were modified. Ensure that the tests cover all relevant changes. ✅" >> $GITHUB_ENV
           else
             echo "comment_message=⚠️ No test files modified. Please ensure that changes are properly tested. ⚠️" >> $GITHUB_ENV
           fi