|
@@ -5,7 +5,7 @@
|
|
|
version: 2.1
|
|
version: 2.1
|
|
|
|
|
|
|
|
commands:
|
|
commands:
|
|
|
- build-common:
|
|
|
|
|
|
|
+ build-py-38-common:
|
|
|
parameters:
|
|
parameters:
|
|
|
go_arch:
|
|
go_arch:
|
|
|
# amd64 or arm64
|
|
# amd64 or arm64
|
|
@@ -16,100 +16,29 @@ commands:
|
|
|
steps:
|
|
steps:
|
|
|
- checkout
|
|
- checkout
|
|
|
|
|
|
|
|
- # Download and cache dependencies
|
|
|
|
|
- # - restore_cache:
|
|
|
|
|
- # keys:
|
|
|
|
|
- # - v1-dependencies-{{ checksum "requirements.txt" }}
|
|
|
|
|
- # # fallback to using the latest cache if no exact match is found
|
|
|
|
|
- # - v1-dependencies-
|
|
|
|
|
-
|
|
|
|
|
- # - run:
|
|
|
|
|
- # name: install dependencies
|
|
|
|
|
- # command: |
|
|
|
|
|
- # python -m venv venv
|
|
|
|
|
- # . venv/bin/activate
|
|
|
|
|
- # pip install -r requirements.txt
|
|
|
|
|
-
|
|
|
|
|
- # - save_cache:
|
|
|
|
|
- # paths:
|
|
|
|
|
- # - ./venv
|
|
|
|
|
- # key: v1-dependencies-{{ checksum "requirements.txt" }}
|
|
|
|
|
-
|
|
|
|
|
- # Revert to traditional pip install when build refactored, in the meantime:
|
|
|
|
|
- run:
|
|
- run:
|
|
|
- name: refresh sources
|
|
|
|
|
|
|
+ name: compile
|
|
|
command: |
|
|
command: |
|
|
|
- if [ ! -d /usr/share/hue ]; then
|
|
|
|
|
- # Ubuntu 20.04 ARM64
|
|
|
|
|
- sudo mkdir -p /usr/share/hue
|
|
|
|
|
- sudo chown -R $USER:$USER /usr/share/hue
|
|
|
|
|
- fi
|
|
|
|
|
-
|
|
|
|
|
- rm -rf /usr/share/hue/desktop/core/src/desktop
|
|
|
|
|
- mkdir -p /usr/share/hue/desktop/core/src/
|
|
|
|
|
- cp -r desktop/core/src/desktop /usr/share/hue/desktop/core/src/desktop
|
|
|
|
|
-
|
|
|
|
|
- for lib in `ls desktop/libs`
|
|
|
|
|
- do
|
|
|
|
|
- rm -rf /usr/share/hue/desktop/libs/$lib/src/$lib
|
|
|
|
|
- mkdir -p /usr/share/hue/desktop/libs/$lib/src/
|
|
|
|
|
- cp -r desktop/libs/$lib/src/$lib /usr/share/hue/desktop/libs/$lib/src/$lib
|
|
|
|
|
- done
|
|
|
|
|
- for lib in `ls apps | grep -v Makefile | grep -v logs`
|
|
|
|
|
- do
|
|
|
|
|
- rm -rf /usr/share/hue/apps/$lib/src/$lib
|
|
|
|
|
- mkdir -p /usr/share/hue/apps/$lib/src/
|
|
|
|
|
- cp -r apps/$lib/src/$lib /usr/share/hue/apps/$lib/src/$lib
|
|
|
|
|
- done
|
|
|
|
|
|
|
|
|
|
- rm -rf /usr/share/hue/tools
|
|
|
|
|
- cp -r tools /usr/share/hue
|
|
|
|
|
-
|
|
|
|
|
- rm -f /usr/share/hue/desktop/conf/*.ini
|
|
|
|
|
- mkdir -p /usr/share/hue/desktop/conf/
|
|
|
|
|
- cp desktop/conf/pseudo-distributed.ini.tmpl /usr/share/hue/desktop/conf/pseudo-distributed.ini
|
|
|
|
|
-
|
|
|
|
|
- # ini configuration tweaks
|
|
|
|
|
- ## Very slow if on, cuts time in two and does not skip tests
|
|
|
|
|
- sed -i 's/## has_iam_detection=true/ has_iam_detection=false/g' /usr/share/hue/desktop/conf/pseudo-distributed.ini
|
|
|
|
|
-
|
|
|
|
|
- # build Hue when using plain Ubuntu image
|
|
|
|
|
- if [ ! -f /usr/share/hue/build/env/bin/hue ]; then
|
|
|
|
|
- sudo add-apt-repository --yes ppa:deadsnakes/ppa
|
|
|
|
|
|
|
+ if [ -d /usr/share/hue ]; then
|
|
|
|
|
+ # Running in gethue/hue Docker container
|
|
|
|
|
+ apt-get update
|
|
|
|
|
+ apt-get install -y python3.8-dev python3.8-venv python3.8-distutils libsnappy-dev # This should not be needed as some point
|
|
|
|
|
+ curl -sL https://bootstrap.pypa.io/get-pip.py | python3.8
|
|
|
|
|
+ else
|
|
|
|
|
+ sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime
|
|
|
|
|
+ export DEBIAN_FRONTEND=noninteractive
|
|
|
sudo apt-get update
|
|
sudo apt-get update
|
|
|
- DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true TZ=UTC sudo apt-get install -y --no-install-recommends python2.7-dev libsnappy-dev asciidoc git ant gcc g++ libffi-dev libkrb5-dev libmysqlclient-dev libsasl2-dev libsasl2-modules-gssapi-mit libsqlite3-dev libssl-dev libxml2-dev libxslt-dev make maven libldap2-dev python-setuptools libgmp3-dev
|
|
|
|
|
- # see https://docs.gethue.com/administrator/installation/dependencies/#mysql--mariadb
|
|
|
|
|
- git config user.email "hue+circleci@example.com"
|
|
|
|
|
- git config user.name "Hue CircleCI"
|
|
|
|
|
- git cherry-pick 7a9100d4a7f38eaef7bd4bd7c715ac1f24a969a8
|
|
|
|
|
- git cherry-pick e67c1105b85b815346758ef1b9cd714dd91d7ea3
|
|
|
|
|
- git clean -fdx
|
|
|
|
|
- make apps
|
|
|
|
|
- cp -r build /usr/share/hue/
|
|
|
|
|
|
|
+ sudo apt-get install -y gcc g++ build-essential python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev
|
|
|
|
|
+ curl -sL https://deb.nodesource.com/setup_14.x | sudo bash - && sudo apt-get install -y nodejs
|
|
|
|
|
+ curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python3.8
|
|
|
|
|
+ sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
|
|
|
fi
|
|
fi
|
|
|
|
|
|
|
|
- cd /usr/share/hue
|
|
|
|
|
- # make npm-install # Not available
|
|
|
|
|
- cp ~/repo/babel.config.js .
|
|
|
|
|
- cp ~/repo/tsconfig.json .
|
|
|
|
|
- cp ~/repo/jest.config.js .
|
|
|
|
|
- cp ~/repo/.pylintrc .
|
|
|
|
|
- cp ~/repo/.stylelintrc .
|
|
|
|
|
-
|
|
|
|
|
- cp ~/repo/webpack.config*.js .
|
|
|
|
|
- rm -f package.json
|
|
|
|
|
- cp ~/repo/package.json .
|
|
|
|
|
- rm -f package-lock.json
|
|
|
|
|
- cp ~/repo/package-lock.json .
|
|
|
|
|
- npm install
|
|
|
|
|
- npm run webpack
|
|
|
|
|
- npm run webpack-login
|
|
|
|
|
- npm run webpack-workers
|
|
|
|
|
- ./build/env/bin/hue collectstatic --noinput
|
|
|
|
|
-
|
|
|
|
|
- cp -r ~/repo/docs .
|
|
|
|
|
|
|
+ export PYTHON_VER=python3.8
|
|
|
|
|
+ export ROOT=$PWD
|
|
|
|
|
+ make apps
|
|
|
|
|
|
|
|
- # Run documentation lint
|
|
|
|
|
- run:
|
|
- run:
|
|
|
name: run documentation lints
|
|
name: run documentation lints
|
|
|
command: |
|
|
command: |
|
|
@@ -130,86 +59,35 @@ commands:
|
|
|
./tools/ci/check_for_website_dead_links.sh docs/docs-site
|
|
./tools/ci/check_for_website_dead_links.sh docs/docs-site
|
|
|
# ./tools/ci/check_for_website_dead_links.sh docs/gethue
|
|
# ./tools/ci/check_for_website_dead_links.sh docs/gethue
|
|
|
|
|
|
|
|
- # Run code lints
|
|
|
|
|
- run:
|
|
- run:
|
|
|
name: run python lints
|
|
name: run python lints
|
|
|
command: |
|
|
command: |
|
|
|
- cd ~/repo
|
|
|
|
|
-
|
|
|
|
|
- /usr/share/hue/build/env/bin/pip install pylint==1.7.5 pylint-django==0.7.2 configparser==4.0.2
|
|
|
|
|
- ./tools/ci/check_for_python_lint.sh /usr/share/hue
|
|
|
|
|
|
|
+ ./build/env/bin/pip install pylint==2.5.3 pylint-django==2.3.0 configparser==5.3.0
|
|
|
|
|
+ ./tools/ci/check_for_python_lint.sh
|
|
|
|
|
|
|
|
- # Run tests
|
|
|
|
|
- run:
|
|
- run:
|
|
|
- name: run python API tests
|
|
|
|
|
|
|
+ name: run tests
|
|
|
command: |
|
|
command: |
|
|
|
- cd /usr/share/hue
|
|
|
|
|
-
|
|
|
|
|
PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
|
|
PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
|
|
|
|
|
|
|
|
- store_artifacts:
|
|
- store_artifacts:
|
|
|
path: test-reports
|
|
path: test-reports
|
|
|
destination: test-reports
|
|
destination: test-reports
|
|
|
|
|
|
|
|
- build-py-38-common:
|
|
|
|
|
- steps:
|
|
|
|
|
- - checkout
|
|
|
|
|
-
|
|
|
|
|
- - run:
|
|
|
|
|
- name: compile
|
|
|
|
|
- command: |
|
|
|
|
|
-
|
|
|
|
|
- if [ -d /usr/share/hue ]; then
|
|
|
|
|
- # Running in gethue/hue Docker container
|
|
|
|
|
- apt-get update
|
|
|
|
|
- apt-get install -y python3.8-dev python3.8-venv python3.8-distutils libsnappy-dev # This should not be needed as some point
|
|
|
|
|
- curl -sL https://bootstrap.pypa.io/get-pip.py | python3.8
|
|
|
|
|
- else
|
|
|
|
|
- sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime
|
|
|
|
|
- export DEBIAN_FRONTEND=noninteractive
|
|
|
|
|
- sudo apt-get update
|
|
|
|
|
- sudo apt-get install -y gcc g++ build-essential python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev
|
|
|
|
|
- curl -sL https://deb.nodesource.com/setup_14.x | sudo bash - && sudo apt-get install -y nodejs
|
|
|
|
|
- curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python3.8
|
|
|
|
|
- sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
|
|
|
|
|
- fi
|
|
|
|
|
-
|
|
|
|
|
- export PYTHON_VER=python3.8
|
|
|
|
|
- export ROOT=$PWD
|
|
|
|
|
- make apps
|
|
|
|
|
-
|
|
|
|
|
- - run:
|
|
|
|
|
- name: run python lints
|
|
|
|
|
- command: |
|
|
|
|
|
- ./build/env/bin/pip install pylint==2.5.3 pylint-django==2.3.0 configparser==5.3.0
|
|
|
|
|
- ./tools/ci/check_for_python_lint.sh
|
|
|
|
|
-
|
|
|
|
|
- - run:
|
|
|
|
|
- name: run tests
|
|
|
|
|
- command: |
|
|
|
|
|
- PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
|
|
|
|
|
-
|
|
|
|
|
|
|
|
|
|
jobs:
|
|
jobs:
|
|
|
- build:
|
|
|
|
|
|
|
+ build-py-38:
|
|
|
docker:
|
|
docker:
|
|
|
- # specify the version you desire here
|
|
|
|
|
- # - image: circleci/python:3.8.0
|
|
|
|
|
- - image: gethue/hue:latest-py2
|
|
|
|
|
-
|
|
|
|
|
- # Specify service dependencies here if necessary
|
|
|
|
|
- # CircleCI maintains a library of pre-built images
|
|
|
|
|
- # documented at https://circleci.com/docs/2.0/circleci-images/
|
|
|
|
|
- # - image: circleci/postgres:9.4
|
|
|
|
|
|
|
+ - image: gethue/hue:latest-py2 # Should be circleci/python:3.6 at some point
|
|
|
|
|
|
|
|
working_directory: ~/repo
|
|
working_directory: ~/repo
|
|
|
|
|
|
|
|
steps:
|
|
steps:
|
|
|
- - build-common:
|
|
|
|
|
|
|
+ - build-py-38-common:
|
|
|
go_arch: "amd64"
|
|
go_arch: "amd64"
|
|
|
hugo_arch: "64bit"
|
|
hugo_arch: "64bit"
|
|
|
|
|
|
|
|
- build-arm64:
|
|
|
|
|
|
|
+ build-py-38-arm64:
|
|
|
machine:
|
|
machine:
|
|
|
image: ubuntu-2004:202101-01
|
|
image: ubuntu-2004:202101-01
|
|
|
resource_class: arm.medium
|
|
resource_class: arm.medium
|
|
@@ -217,70 +95,12 @@ jobs:
|
|
|
working_directory: ~/repo
|
|
working_directory: ~/repo
|
|
|
|
|
|
|
|
steps:
|
|
steps:
|
|
|
- - build-common:
|
|
|
|
|
|
|
+ - build-py-38-common:
|
|
|
go_arch: "arm64"
|
|
go_arch: "arm64"
|
|
|
hugo_arch: "ARM64"
|
|
hugo_arch: "ARM64"
|
|
|
|
|
|
|
|
- commit:
|
|
|
|
|
- docker:
|
|
|
|
|
- - image: circleci/python:3.8.0
|
|
|
|
|
-
|
|
|
|
|
- working_directory: ~/repo
|
|
|
|
|
-
|
|
|
|
|
- steps:
|
|
|
|
|
- - checkout
|
|
|
|
|
-
|
|
|
|
|
- - add_ssh_keys:
|
|
|
|
|
- fingerprints:
|
|
|
|
|
- - "8c:68:29:28:04:80:38:31:c0:59:d9:3d:65:3e:b7:8c"
|
|
|
|
|
-
|
|
|
|
|
- - run:
|
|
|
|
|
- name: push to master
|
|
|
|
|
- command: |
|
|
|
|
|
- git push origin HEAD:master
|
|
|
|
|
-
|
|
|
|
|
- build-py-38:
|
|
|
|
|
- docker:
|
|
|
|
|
- - image: gethue/hue:latest-py2 # Should be circleci/python:3.6 at some point
|
|
|
|
|
-
|
|
|
|
|
- working_directory: ~/repo
|
|
|
|
|
-
|
|
|
|
|
- steps:
|
|
|
|
|
- - build-py-38-common
|
|
|
|
|
-
|
|
|
|
|
- build-py-38-arm64:
|
|
|
|
|
- machine:
|
|
|
|
|
- image: ubuntu-2004:202101-01
|
|
|
|
|
- resource_class: arm.medium
|
|
|
|
|
-
|
|
|
|
|
- working_directory: ~/repo
|
|
|
|
|
-
|
|
|
|
|
- steps:
|
|
|
|
|
- - build-py-38-common
|
|
|
|
|
-
|
|
|
|
|
workflows:
|
|
workflows:
|
|
|
version: 2
|
|
version: 2
|
|
|
- run-tests-and-rebase-master:
|
|
|
|
|
- jobs:
|
|
|
|
|
- - build:
|
|
|
|
|
- filters:
|
|
|
|
|
- branches:
|
|
|
|
|
- ignore:
|
|
|
|
|
- - master
|
|
|
|
|
- - py3-ci
|
|
|
|
|
- - build-arm64:
|
|
|
|
|
- filters:
|
|
|
|
|
- branches:
|
|
|
|
|
- ignore:
|
|
|
|
|
- - master
|
|
|
|
|
- - py3-ci
|
|
|
|
|
- - commit:
|
|
|
|
|
- requires:
|
|
|
|
|
- - build
|
|
|
|
|
- filters:
|
|
|
|
|
- branches:
|
|
|
|
|
- only:
|
|
|
|
|
- - /.*ci-commit-master.*/
|
|
|
|
|
build-python3:
|
|
build-python3:
|
|
|
jobs:
|
|
jobs:
|
|
|
- build-py-38:
|
|
- build-py-38:
|