# Python CircleCI 2.0 configuration file # # Check https://circleci.com/docs/2.0/language-python/ for more details # version: 2.1 commands: build-common: parameters: go_arch: # amd64 or arm64 type: string hugo_arch: # 64bit (AMD64) or ARM64 type: string steps: - checkout # Download and cache dependencies # - restore_cache: # keys: # - v1-dependencies-{{ checksum "requirements.txt" }} # # fallback to using the latest cache if no exact match is found # - v1-dependencies- # - run: # name: install dependencies # command: | # python -m venv venv # . venv/bin/activate # pip install -r requirements.txt # - save_cache: # paths: # - ./venv # key: v1-dependencies-{{ checksum "requirements.txt" }} # Revert to traditional pip install when build refactored, in the meantime: - run: name: refresh sources command: | if [ ! -d /usr/share/hue ]; then # Ubuntu 20.04 ARM64 sudo mkdir -p /usr/share/hue sudo chown -R $USER:$USER /usr/share/hue fi rm -rf /usr/share/hue/desktop/core/src/desktop mkdir -p /usr/share/hue/desktop/core/src/ cp -r desktop/core/src/desktop /usr/share/hue/desktop/core/src/desktop for lib in `ls desktop/libs` do rm -rf /usr/share/hue/desktop/libs/$lib/src/$lib mkdir -p /usr/share/hue/desktop/libs/$lib/src/ cp -r desktop/libs/$lib/src/$lib /usr/share/hue/desktop/libs/$lib/src/$lib done for lib in `ls apps | grep -v Makefile | grep -v logs` do rm -rf /usr/share/hue/apps/$lib/src/$lib mkdir -p /usr/share/hue/apps/$lib/src/ cp -r apps/$lib/src/$lib /usr/share/hue/apps/$lib/src/$lib done rm -rf /usr/share/hue/tools cp -r tools /usr/share/hue rm -f /usr/share/hue/desktop/conf/*.ini mkdir -p /usr/share/hue/desktop/conf/ cp desktop/conf/pseudo-distributed.ini.tmpl /usr/share/hue/desktop/conf/pseudo-distributed.ini # ini configuration tweaks ## Very slow if on, cuts time in two and does not skip tests sed -i 's/## has_iam_detection=true/ has_iam_detection=false/g' /usr/share/hue/desktop/conf/pseudo-distributed.ini # build Hue when using plain Ubuntu image if [ ! -f /usr/share/hue/build/env/bin/hue ]; then sudo add-apt-repository --yes ppa:deadsnakes/ppa sudo apt-get update DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true TZ=UTC sudo apt-get install -y --no-install-recommends python2.7-dev libsnappy-dev asciidoc git ant gcc g++ libffi-dev libkrb5-dev libmysqlclient-dev libsasl2-dev libsasl2-modules-gssapi-mit libsqlite3-dev libssl-dev libxml2-dev libxslt-dev make maven libldap2-dev python-setuptools libgmp3-dev # see https://docs.gethue.com/administrator/installation/dependencies/#mysql--mariadb git config user.email "hue+circleci@example.com" git config user.name "Hue CircleCI" git cherry-pick 7a9100d4a7f38eaef7bd4bd7c715ac1f24a969a8 git cherry-pick e67c1105b85b815346758ef1b9cd714dd91d7ea3 git clean -fdx make apps cp -r build /usr/share/hue/ fi cd /usr/share/hue # make npm-install # Not available cp ~/repo/babel.config.js . cp ~/repo/tsconfig.json . cp ~/repo/jest.config.js . cp ~/repo/.pylintrc . cp ~/repo/.stylelintrc . cp ~/repo/webpack.config*.js . rm -f package.json cp ~/repo/package.json . rm -f package-lock.json cp ~/repo/package-lock.json . npm install npm i eslint-plugin-jest@latest --save-dev # Seems to not be found otherwise npm run webpack npm run webpack-login npm run webpack-workers ./build/env/bin/hue collectstatic --noinput cp -r ~/repo/docs . # Run documentation lint - run: name: run documentation lints command: | cd ~/repo # Installs to move to image building curl -O https://dl.google.com/go/go1.17.linux-<< parameters.go_arch >>.tar.gz tar -xvf go1.17.linux-<< parameters.go_arch >>.tar.gz export GO111MODULE=on go/bin/go get -u github.com/raviqqe/muffet@v1.5.7 curl --output hugo_0.69.0_Linux.tar.gz -L https://github.com/gohugoio/hugo/releases/download/v0.69.0/hugo_0.69.0_Linux-<< parameters.hugo_arch >>.tar.gz tar -xvf hugo_0.69.0_Linux.tar.gz export PATH=$PWD:$HOME/go/bin:$PATH # Trigger linting if documentation changes ./tools/ci/check_for_website_dead_links.sh docs/docs-site # ./tools/ci/check_for_website_dead_links.sh docs/gethue # Run code lints - run: name: run python lints command: | cd ~/repo /usr/share/hue/build/env/bin/pip install pylint==1.7.5 pylint-django==0.7.2 configparser==4.0.2 ./tools/ci/check_for_python_lint.sh /usr/share/hue - run: name: run commit title format check command: | cd ~/repo ./tools/ci/check_for_commit_message.sh - run: name: run js lint command: | cd /usr/share/hue cp ~/repo/.prettierrc . cp ~/repo/.eslint* . cp ~/repo/tools . -r npm run lint - run: name: run style lint command: | cd /usr/share/hue npm run style-lint - run: name: run npm license checker command: | cd /usr/share/hue npm run check-license - run: name: run npm absolute path detection command: | cd /usr/share/hue npm run check-absolute-paths # Run tests - run: name: run python API tests command: | cd /usr/share/hue PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover - run: name: run js tests command: | cd /usr/share/hue # https://jestjs.io/docs/en/troubleshooting.html#tests-are-extremely-slow-on-docker-andor-continuous-integration-ci-server sed -i 's/"test": "jest"/"test": "jest --runInBand"/g' package.json npm run test - store_artifacts: path: test-reports destination: test-reports build-py-38-common: steps: - checkout - run: name: compile command: | if [ -d /usr/share/hue ]; then # Running in gethue/hue Docker container apt-get install -y python3.8-dev python3.8-venv python3.8-distutils libsnappy-dev # This should not be needed as some point curl -sL https://bootstrap.pypa.io/get-pip.py | python3.8 else sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y gcc g++ build-essential python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev curl -sL https://deb.nodesource.com/setup_14.x | sudo bash - && sudo apt-get install -y nodejs curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python3.8 sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl fi export PYTHON_VER=python3.8 export ROOT=$PWD make apps - run: name: run tests command: | PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover jobs: build: docker: # specify the version you desire here # - image: circleci/python:3.8.0 - image: gethue/hue:latest-py2 # Specify service dependencies here if necessary # CircleCI maintains a library of pre-built images # documented at https://circleci.com/docs/2.0/circleci-images/ # - image: circleci/postgres:9.4 working_directory: ~/repo steps: - build-common: go_arch: "amd64" hugo_arch: "64bit" build-arm64: machine: image: ubuntu-2004:202101-01 resource_class: arm.medium working_directory: ~/repo steps: - build-common: go_arch: "arm64" hugo_arch: "ARM64" commit: docker: - image: circleci/python:3.8.0 working_directory: ~/repo steps: - checkout - add_ssh_keys: fingerprints: - "8c:68:29:28:04:80:38:31:c0:59:d9:3d:65:3e:b7:8c" - run: name: push to master command: | git push origin HEAD:master build-py-38: docker: - image: gethue/hue:latest-py2 # Should be circleci/python:3.6 at some point working_directory: ~/repo steps: - build-py-38-common build-py-38-arm64: machine: image: ubuntu-2004:202101-01 resource_class: arm.medium working_directory: ~/repo steps: - build-py-38-common workflows: version: 2 run-tests-and-rebase-master: jobs: - build: filters: branches: ignore: - master - py3-ci - build-arm64: filters: branches: ignore: - master - py3-ci - commit: requires: - build filters: branches: only: - /.*ci-commit-master.*/ build-python3: jobs: - build-py-38: filters: branches: ignore: - master - build-py-38-arm64: filters: branches: ignore: - master