| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337 |
- # Python CircleCI 2.0 configuration file
- #
- # Check https://circleci.com/docs/2.0/language-python/ for more details
- #
- version: 2.1
- commands:
- build-common:
- parameters:
- go_arch:
- # amd64 or arm64
- type: string
- hugo_arch:
- # 64bit (AMD64) or ARM64
- type: string
- steps:
- - checkout
- # Download and cache dependencies
- # - restore_cache:
- # keys:
- # - v1-dependencies-{{ checksum "requirements.txt" }}
- # # fallback to using the latest cache if no exact match is found
- # - v1-dependencies-
- # - run:
- # name: install dependencies
- # command: |
- # python -m venv venv
- # . venv/bin/activate
- # pip install -r requirements.txt
- # - save_cache:
- # paths:
- # - ./venv
- # key: v1-dependencies-{{ checksum "requirements.txt" }}
- # Revert to traditional pip install when build refactored, in the meantime:
- - run:
- name: refresh sources
- command: |
- if [ ! -d /usr/share/hue ]; then
- # Ubuntu 20.04 ARM64
- sudo mkdir -p /usr/share/hue
- sudo chown -R $USER:$USER /usr/share/hue
- fi
- rm -rf /usr/share/hue/desktop/core/src/desktop
- mkdir -p /usr/share/hue/desktop/core/src/
- cp -r desktop/core/src/desktop /usr/share/hue/desktop/core/src/desktop
- for lib in `ls desktop/libs`
- do
- rm -rf /usr/share/hue/desktop/libs/$lib/src/$lib
- mkdir -p /usr/share/hue/desktop/libs/$lib/src/
- cp -r desktop/libs/$lib/src/$lib /usr/share/hue/desktop/libs/$lib/src/$lib
- done
- for lib in `ls apps | grep -v Makefile | grep -v logs`
- do
- rm -rf /usr/share/hue/apps/$lib/src/$lib
- mkdir -p /usr/share/hue/apps/$lib/src/
- cp -r apps/$lib/src/$lib /usr/share/hue/apps/$lib/src/$lib
- done
- rm -rf /usr/share/hue/tools
- cp -r tools /usr/share/hue
- rm -f /usr/share/hue/desktop/conf/*.ini
- mkdir -p /usr/share/hue/desktop/conf/
- cp desktop/conf/pseudo-distributed.ini.tmpl /usr/share/hue/desktop/conf/pseudo-distributed.ini
- # ini configuration tweaks
- ## Very slow if on, cuts time in two and does not skip tests
- sed -i 's/## has_iam_detection=true/ has_iam_detection=false/g' /usr/share/hue/desktop/conf/pseudo-distributed.ini
- # build Hue when using plain Ubuntu image
- if [ ! -f /usr/share/hue/build/env/bin/hue ]; then
- sudo add-apt-repository --yes ppa:deadsnakes/ppa
- sudo apt-get update
- DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true TZ=UTC sudo apt-get install -y --no-install-recommends python2.7-dev libsnappy-dev asciidoc git ant gcc g++ libffi-dev libkrb5-dev libmysqlclient-dev libsasl2-dev libsasl2-modules-gssapi-mit libsqlite3-dev libssl-dev libxml2-dev libxslt-dev make maven libldap2-dev python-setuptools libgmp3-dev
- # see https://docs.gethue.com/administrator/installation/dependencies/#mysql--mariadb
- git config user.email "hue+circleci@example.com"
- git config user.name "Hue CircleCI"
- git cherry-pick 7a9100d4a7f38eaef7bd4bd7c715ac1f24a969a8
- git cherry-pick e67c1105b85b815346758ef1b9cd714dd91d7ea3
- git clean -fdx
- make apps
- cp -r build /usr/share/hue/
- fi
- cd /usr/share/hue
- # make npm-install # Not available
- cp ~/repo/babel.config.js .
- cp ~/repo/tsconfig.json .
- cp ~/repo/jest.config.js .
- cp ~/repo/.pylintrc .
- cp ~/repo/.stylelintrc .
- cp ~/repo/webpack.config*.js .
- rm -f package.json
- cp ~/repo/package.json .
- rm -f package-lock.json
- cp ~/repo/package-lock.json .
- npm install
- npm i eslint-plugin-jest@latest --save-dev # Seems to not be found otherwise
- npm run webpack
- npm run webpack-login
- npm run webpack-workers
- ./build/env/bin/hue collectstatic --noinput
- cp -r ~/repo/docs .
- # Run documentation lint
- - run:
- name: run documentation lints
- command: |
- cd ~/repo
- # Installs to move to image building
- curl -O https://dl.google.com/go/go1.17.linux-<< parameters.go_arch >>.tar.gz
- tar -xvf go1.17.linux-<< parameters.go_arch >>.tar.gz
- export GO111MODULE=on
- go/bin/go get -u github.com/raviqqe/muffet@v1.5.7
- curl --output hugo_0.69.0_Linux.tar.gz -L https://github.com/gohugoio/hugo/releases/download/v0.69.0/hugo_0.69.0_Linux-<< parameters.hugo_arch >>.tar.gz
- tar -xvf hugo_0.69.0_Linux.tar.gz
- export PATH=$PWD:$HOME/go/bin:$PATH
- # Trigger linting if documentation changes
- ./tools/ci/check_for_website_dead_links.sh docs/docs-site
- # ./tools/ci/check_for_website_dead_links.sh docs/gethue
- # Run code lints
- - run:
- name: run python lints
- command: |
- cd ~/repo
- /usr/share/hue/build/env/bin/pip install pylint==1.7.5 pylint-django==0.7.2 configparser==4.0.2
- ./tools/ci/check_for_python_lint.sh /usr/share/hue
- - run:
- name: run commit title format check
- command: |
- cd ~/repo
- ./tools/ci/check_for_commit_message.sh
- - run:
- name: run js lint
- command: |
- cd /usr/share/hue
- cp ~/repo/.prettierrc .
- cp ~/repo/.eslint* .
- cp ~/repo/tools . -r
- npm run lint
- - run:
- name: run style lint
- command: |
- cd /usr/share/hue
- npm run style-lint
- - run:
- name: run npm license checker
- command: |
- cd /usr/share/hue
- npm run check-license
- - run:
- name: run npm absolute path detection
- command: |
- cd /usr/share/hue
- npm run check-absolute-paths
- # Run tests
- - run:
- name: run python API tests
- command: |
- cd /usr/share/hue
- PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
- - run:
- name: run js tests
- command: |
- cd /usr/share/hue
- # https://jestjs.io/docs/en/troubleshooting.html#tests-are-extremely-slow-on-docker-andor-continuous-integration-ci-server
- sed -i 's/"test": "jest"/"test": "jest --runInBand"/g' package.json
- npm run test
- - store_artifacts:
- path: test-reports
- destination: test-reports
- build-py-38-common:
- steps:
- - checkout
- - run:
- name: compile
- command: |
- if [ -d /usr/share/hue ]; then
- # Running in gethue/hue Docker container
- apt-get install -y python3.8-dev python3.8-venv python3.8-distutils libsnappy-dev # This should not be needed as some point
- curl -sL https://bootstrap.pypa.io/get-pip.py | python3.8
- else
- sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime
- export DEBIAN_FRONTEND=noninteractive
- sudo apt-get update
- sudo apt-get install -y gcc g++ build-essential python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev
- curl -sL https://deb.nodesource.com/setup_14.x | sudo bash - && sudo apt-get install -y nodejs
- curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python3.8
- sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
- fi
- export PYTHON_VER=python3.8
- export ROOT=$PWD
- make apps
- - run:
- name: run tests
- command: |
- PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
- jobs:
- build:
- docker:
- # specify the version you desire here
- # - image: circleci/python:3.8.0
- - image: gethue/hue:latest-py2
- # Specify service dependencies here if necessary
- # CircleCI maintains a library of pre-built images
- # documented at https://circleci.com/docs/2.0/circleci-images/
- # - image: circleci/postgres:9.4
- working_directory: ~/repo
- steps:
- - build-common:
- go_arch: "amd64"
- hugo_arch: "64bit"
- build-arm64:
- machine:
- image: ubuntu-2004:202101-01
- resource_class: arm.medium
- working_directory: ~/repo
- steps:
- - build-common:
- go_arch: "arm64"
- hugo_arch: "ARM64"
- commit:
- docker:
- - image: circleci/python:3.8.0
- working_directory: ~/repo
- steps:
- - checkout
- - add_ssh_keys:
- fingerprints:
- - "8c:68:29:28:04:80:38:31:c0:59:d9:3d:65:3e:b7:8c"
- - run:
- name: push to master
- command: |
- git push origin HEAD:master
- build-py-38:
- docker:
- - image: gethue/hue:latest-py2 # Should be circleci/python:3.6 at some point
- working_directory: ~/repo
- steps:
- - build-py-38-common
- build-py-38-arm64:
- machine:
- image: ubuntu-2004:202101-01
- resource_class: arm.medium
- working_directory: ~/repo
- steps:
- - build-py-38-common
- workflows:
- version: 2
- run-tests-and-rebase-master:
- jobs:
- - build:
- filters:
- branches:
- ignore:
- - master
- - py3-ci
- - build-arm64:
- filters:
- branches:
- ignore:
- - master
- - py3-ci
- - commit:
- requires:
- - build
- filters:
- branches:
- only:
- - /.*ci-commit-master.*/
- build-python3:
- jobs:
- - build-py-38:
- filters:
- branches:
- ignore:
- - master
- - build-py-38-arm64:
- filters:
- branches:
- ignore:
- - master
|