Эх сурвалжийг харах

CircleCI: Add build jobs that run on Linux ARM64 (#2834)

* Add jobs to verify the PRs on Linux ARM64 at CircleCI

The ARM64 jobs run on plain Ubuntu 20.04, i.e. not on gethue/hue image
and thus requires different 'install prerequisites' step.

* Remove duplicated Ubuntu package installations
* Use virtualenv instead of venv
* Do not upgrade pip and setuptools
* Do not add ~/.local/bin to $PATH
* Use $(PYTHON_VER) instead of hardcoded 'python3.8'

Signed-off-by: Martin Tzvetanov Grigorov <mgrigorov@apache.org>
Co-authored-by: Harsh Gupta <42064744+Harshg999@users.noreply.github.com>
Martin Grigorov 3 жил өмнө
parent
commit
42f025c7d0

+ 83 - 52
.circleci/config.yml

@@ -39,30 +39,55 @@ commands:
       - run:
           name: refresh sources
           command: |
-            rm -r /usr/share/hue/desktop/core/src/desktop
+            if [ ! -d /usr/share/hue ]; then
+              # Ubuntu 20.04 ARM64
+              sudo mkdir -p /usr/share/hue
+              sudo chown -R $USER:$USER /usr/share/hue
+            fi
+
+            rm -rf /usr/share/hue/desktop/core/src/desktop
+            mkdir -p /usr/share/hue/desktop/core/src/
             cp -r desktop/core/src/desktop /usr/share/hue/desktop/core/src/desktop
 
             for lib in `ls desktop/libs`
             do
-              rm -r /usr/share/hue/desktop/libs/$lib/src/$lib
+              rm -rf /usr/share/hue/desktop/libs/$lib/src/$lib
+              mkdir -p /usr/share/hue/desktop/libs/$lib/src/
               cp -r desktop/libs/$lib/src/$lib /usr/share/hue/desktop/libs/$lib/src/$lib
             done
-            for lib in `ls apps | grep -v Makefile`
+            for lib in `ls apps | grep -v Makefile | grep -v logs`
             do
-              rm -r /usr/share/hue/apps/$lib/src/$lib
+              rm -rf /usr/share/hue/apps/$lib/src/$lib
+              mkdir -p /usr/share/hue/apps/$lib/src/
               cp -r apps/$lib/src/$lib /usr/share/hue/apps/$lib/src/$lib
             done
 
-            rm -r /usr/share/hue/tools
+            rm -rf /usr/share/hue/tools
             cp -r tools /usr/share/hue
 
-            rm /usr/share/hue/desktop/conf/*.ini
+            rm -f /usr/share/hue/desktop/conf/*.ini
+            mkdir -p /usr/share/hue/desktop/conf/
             cp desktop/conf/pseudo-distributed.ini.tmpl /usr/share/hue/desktop/conf/pseudo-distributed.ini
 
             # ini configuration tweaks
             ## Very slow if on, cuts time in two and does not skip tests
             sed -i 's/## has_iam_detection=true/  has_iam_detection=false/g' /usr/share/hue/desktop/conf/pseudo-distributed.ini
 
+            # build Hue when using plain Ubuntu image
+            if [ ! -f /usr/share/hue/build/env/bin/hue ]; then
+              sudo add-apt-repository --yes ppa:deadsnakes/ppa
+              sudo apt-get update
+              DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true TZ=UTC sudo apt-get install -y --no-install-recommends python2.7-dev libsnappy-dev asciidoc git ant gcc g++ libffi-dev libkrb5-dev libmysqlclient-dev libsasl2-dev libsasl2-modules-gssapi-mit libsqlite3-dev libssl-dev libxml2-dev libxslt-dev make maven libldap2-dev python-setuptools libgmp3-dev
+              # see https://docs.gethue.com/administrator/installation/dependencies/#mysql--mariadb
+              git config user.email "hue+circleci@example.com"
+              git config user.name "Hue CircleCI"
+              git cherry-pick 7a9100d4a7f38eaef7bd4bd7c715ac1f24a969a8
+              git cherry-pick e67c1105b85b815346758ef1b9cd714dd91d7ea3
+              git clean -fdx
+              make apps
+              cp -r build /usr/share/hue/
+            fi
+
             cd /usr/share/hue
             # make npm-install # Not available
             cp ~/repo/babel.config.js .
@@ -72,9 +97,9 @@ commands:
             cp ~/repo/.stylelintrc .
 
             cp ~/repo/webpack.config*.js .
-            rm package.json
+            rm -f package.json
             cp ~/repo/package.json .
-            rm package-lock.json
+            rm -f package-lock.json
             cp ~/repo/package-lock.json .
             npm install
             npm i eslint-plugin-jest@latest --save-dev # Seems to not be found otherwise
@@ -175,43 +200,6 @@ commands:
           path: test-reports
           destination: test-reports
 
-  build-py-36-common:
-    steps:
-      - checkout
-
-      # Download and cache dependencies
-      # - restore_cache:
-      #     keys:
-      #       - v1-dependencies-{{ checksum "desktop/core/requirements.txt" }}
-      #       # fallback to using the latest cache if no exact match is found
-      #       - v1-dependencies-
-
-      - run:
-          name: compile
-          command: |
-            apt-get update && apt-get install -y python3.6-dev libsnappy-dev asciidoc # This should not be needed as some point
-
-            export PYTHON_VER=python3.6
-            make apps
-
-            make prod
-
-      - run:
-          name: run tests
-          command: |
-            PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
-
-      # - run:
-      #     name: install dependencies
-      #     command: |
-      #       python -m venv venv
-      #       make apps
-
-      # - save_cache:
-      #     paths:
-      #       - ./build/venv
-      #     key: v1-dependencies-{{ checksum "esktop/core/requirements.txt" }}
-
   build-py-38-common:
     steps:
       - checkout
@@ -219,7 +207,20 @@ commands:
       - run:
           name: compile
           command: |
-            apt-get install -y python3.8-dev python3.8-venv python3.8-distutils libsnappy-dev # This should not be needed as some point
+
+            if [ -d /usr/share/hue ]; then
+              # Running in gethue/hue Docker container
+              apt-get install -y python3.8-dev python3.8-venv python3.8-distutils libsnappy-dev # This should not be needed as some point
+              curl -sL https://bootstrap.pypa.io/get-pip.py | python3.8
+            else
+              sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime
+              export DEBIAN_FRONTEND=noninteractive
+              sudo apt-get update
+              sudo apt-get install -y gcc g++ build-essential python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev  libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev
+              curl -sL https://deb.nodesource.com/setup_14.x | sudo bash - && sudo apt-get install -y nodejs
+              curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python3.8
+              sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
+            fi
 
             export PYTHON_VER=python3.8
             make apps
@@ -249,6 +250,18 @@ jobs:
           go_arch: "amd64"
           hugo_arch: "64bit"
 
+  build-arm64:
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: arm.medium
+
+    working_directory: ~/repo
+
+    steps:
+      - build-common:
+          go_arch: "arm64"
+          hugo_arch: "ARM64"
+
   commit:
     docker:
       - image: circleci/python:3.8.0
@@ -267,19 +280,19 @@ jobs:
           command: |
             git push origin HEAD:master
 
-
-  build-py-36:
+  build-py-38:
     docker:
       - image: gethue/hue:latest-py2 # Should be circleci/python:3.6 at some point
 
     working_directory: ~/repo
 
     steps:
-      - build-py-36-common
+      - build-py-38-common
 
-  build-py-38:
-    docker:
-      - image: gethue/hue:latest-py2 # Should be circleci/python:3.6 at some point
+  build-py-38-arm64:
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: arm.medium
 
     working_directory: ~/repo
 
@@ -296,6 +309,12 @@ workflows:
               ignore:
                 - master
                 - py3-ci
+      - build-arm64:
+          filters:
+            branches:
+              ignore:
+                - master
+                - py3-ci
       - commit:
           requires:
             - build
@@ -303,3 +322,15 @@ workflows:
             branches:
               only:
                 - /.*ci-commit-master.*/
+  build-python3:
+    jobs:
+      - build-py-38:
+          filters:
+            branches:
+              ignore:
+                - master
+      - build-py-38-arm64:
+          filters:
+            branches:
+              ignore:
+                - master

+ 2 - 2
.github/workflows/commitflow-py3.yml

@@ -37,11 +37,11 @@ jobs:
     - name: compile
       run: |
         sudo apt-get update
-        sudo apt-get install -y gcc g++ build-essential python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl sudo libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev libldap2-dev libsnappy-dev libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev # This should not be needed as some point
+        sudo apt-get install -y gcc g++ build-essential python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl sudo libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev  libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev # This should not be needed as some point
         sudo curl -sL https://deb.nodesource.com/setup_14.x | sudo bash - && sudo apt-get install -y nodejs
         sudo curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python3.8
         sudo apt-get install -y python3-setuptools
-        sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl libffi-dev
+        sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
 
         export PYTHON_VER=python3.8
         make apps

+ 1 - 1
Makefile

@@ -138,7 +138,7 @@ ifeq ($(PYTHON_VER),python2.7)
 else ifeq ($(PYTHON_VER),python3.8)
 	@$(SYS_PYTHON) -m pip install --upgrade pip
 	@$(SYS_PIP) install virtualenv
-	@virtualenv $(BLD_DIR_ENV)
+	@virtualenv -p $(PYTHON_VER) $(BLD_DIR_ENV)
 endif
 	@echo "--- Virtual environment $(BLD_DIR_ENV) ready"
 	@touch $@