config.yml 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350
  1. # Python CircleCI 2.0 configuration file
  2. #
  3. # Check https://circleci.com/docs/2.0/language-python/ for more details
  4. #
  5. version: 2.1
  6. commands:
  7. build-common:
  8. parameters:
  9. go_arch:
  10. # amd64 or arm64
  11. type: string
  12. hugo_arch:
  13. # 64bit (AMD64) or ARM64
  14. type: string
  15. steps:
  16. - checkout
  17. # Download and cache dependencies
  18. # - restore_cache:
  19. # keys:
  20. # - v1-dependencies-{{ checksum "requirements.txt" }}
  21. # # fallback to using the latest cache if no exact match is found
  22. # - v1-dependencies-
  23. # - run:
  24. # name: install dependencies
  25. # command: |
  26. # python -m venv venv
  27. # . venv/bin/activate
  28. # pip install -r requirements.txt
  29. # - save_cache:
  30. # paths:
  31. # - ./venv
  32. # key: v1-dependencies-{{ checksum "requirements.txt" }}
  33. # Revert to traditional pip install when build refactored, in the meantime:
  34. - run:
  35. name: refresh sources
  36. command: |
  37. if [ ! -d /usr/share/hue ]; then
  38. # Ubuntu 20.04 ARM64
  39. sudo mkdir -p /usr/share/hue
  40. sudo chown -R $USER:$USER /usr/share/hue
  41. fi
  42. rm -rf /usr/share/hue/desktop/core/src/desktop
  43. mkdir -p /usr/share/hue/desktop/core/src/
  44. cp -r desktop/core/src/desktop /usr/share/hue/desktop/core/src/desktop
  45. for lib in `ls desktop/libs`
  46. do
  47. rm -rf /usr/share/hue/desktop/libs/$lib/src/$lib
  48. mkdir -p /usr/share/hue/desktop/libs/$lib/src/
  49. cp -r desktop/libs/$lib/src/$lib /usr/share/hue/desktop/libs/$lib/src/$lib
  50. done
  51. for lib in `ls apps | grep -v Makefile | grep -v logs`
  52. do
  53. rm -rf /usr/share/hue/apps/$lib/src/$lib
  54. mkdir -p /usr/share/hue/apps/$lib/src/
  55. cp -r apps/$lib/src/$lib /usr/share/hue/apps/$lib/src/$lib
  56. done
  57. rm -rf /usr/share/hue/tools
  58. cp -r tools /usr/share/hue
  59. rm -f /usr/share/hue/desktop/conf/*.ini
  60. mkdir -p /usr/share/hue/desktop/conf/
  61. cp desktop/conf/pseudo-distributed.ini.tmpl /usr/share/hue/desktop/conf/pseudo-distributed.ini
  62. # ini configuration tweaks
  63. ## Very slow if on, cuts time in two and does not skip tests
  64. sed -i 's/## has_iam_detection=true/ has_iam_detection=false/g' /usr/share/hue/desktop/conf/pseudo-distributed.ini
  65. # build Hue when using plain Ubuntu image
  66. if [ ! -f /usr/share/hue/build/env/bin/hue ]; then
  67. sudo add-apt-repository --yes ppa:deadsnakes/ppa
  68. sudo apt-get update
  69. DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true TZ=UTC sudo apt-get install -y --no-install-recommends python2.7-dev libsnappy-dev asciidoc git ant gcc g++ libffi-dev libkrb5-dev libmysqlclient-dev libsasl2-dev libsasl2-modules-gssapi-mit libsqlite3-dev libssl-dev libxml2-dev libxslt-dev make maven libldap2-dev python-setuptools libgmp3-dev
  70. # see https://docs.gethue.com/administrator/installation/dependencies/#mysql--mariadb
  71. git config user.email "hue+circleci@example.com"
  72. git config user.name "Hue CircleCI"
  73. git cherry-pick 7a9100d4a7f38eaef7bd4bd7c715ac1f24a969a8
  74. git cherry-pick e67c1105b85b815346758ef1b9cd714dd91d7ea3
  75. git clean -fdx
  76. make apps
  77. cp -r build /usr/share/hue/
  78. fi
  79. cd /usr/share/hue
  80. # make npm-install # Not available
  81. cp ~/repo/babel.config.js .
  82. cp ~/repo/tsconfig.json .
  83. cp ~/repo/jest.config.js .
  84. cp ~/repo/.pylintrc .
  85. cp ~/repo/.stylelintrc .
  86. cp ~/repo/webpack.config*.js .
  87. rm -f package.json
  88. cp ~/repo/package.json .
  89. rm -f package-lock.json
  90. cp ~/repo/package-lock.json .
  91. npm install
  92. npm run webpack
  93. npm run webpack-login
  94. npm run webpack-workers
  95. ./build/env/bin/hue collectstatic --noinput
  96. cp -r ~/repo/docs .
  97. - run:
  98. name: run commit title format check
  99. command: |
  100. cd ~/repo
  101. ./tools/ci/check_for_commit_message.sh
  102. # Run documentation lint
  103. - run:
  104. name: run documentation lints
  105. command: |
  106. cd ~/repo
  107. # Installs to move to image building
  108. curl -O https://dl.google.com/go/go1.17.linux-<< parameters.go_arch >>.tar.gz
  109. tar -xvf go1.17.linux-<< parameters.go_arch >>.tar.gz
  110. export GO111MODULE=on
  111. go/bin/go get -u github.com/raviqqe/muffet@v1.5.7
  112. curl --output hugo_0.69.0_Linux.tar.gz -L https://github.com/gohugoio/hugo/releases/download/v0.69.0/hugo_0.69.0_Linux-<< parameters.hugo_arch >>.tar.gz
  113. tar -xvf hugo_0.69.0_Linux.tar.gz
  114. export PATH=$PWD:$HOME/go/bin:$PATH
  115. # Trigger linting if documentation changes
  116. ./tools/ci/check_for_website_dead_links.sh docs/docs-site
  117. # ./tools/ci/check_for_website_dead_links.sh docs/gethue
  118. # Run code lints
  119. - run:
  120. name: run python lints
  121. command: |
  122. cd ~/repo
  123. /usr/share/hue/build/env/bin/pip install pylint==1.7.5 pylint-django==0.7.2 configparser==4.0.2
  124. ./tools/ci/check_for_python_lint.sh /usr/share/hue
  125. - run:
  126. name: run js lint
  127. command: |
  128. cd /usr/share/hue
  129. cp ~/repo/.prettierrc .
  130. cp ~/repo/.eslint* .
  131. cp ~/repo/tools . -r
  132. npm run lint
  133. - run:
  134. name: run style lint
  135. command: |
  136. cd /usr/share/hue
  137. npm run style-lint
  138. - run:
  139. name: run npm version checker
  140. command: |
  141. cd /usr/share/hue
  142. npm run check-pinned-versions
  143. - run:
  144. name: run npm license checker
  145. command: |
  146. cd /usr/share/hue
  147. npm run check-license
  148. - run:
  149. name: run npm absolute path detection
  150. command: |
  151. cd /usr/share/hue
  152. npm run check-absolute-paths
  153. # Run tests
  154. - run:
  155. name: run python API tests
  156. command: |
  157. cd /usr/share/hue
  158. PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
  159. - run:
  160. name: run js tests
  161. command: |
  162. cd /usr/share/hue
  163. # https://jestjs.io/docs/en/troubleshooting.html#tests-are-extremely-slow-on-docker-andor-continuous-integration-ci-server
  164. sed -i 's/"test": "jest"/"test": "jest --runInBand"/g' package.json
  165. npm run test
  166. - store_artifacts:
  167. path: test-reports
  168. destination: test-reports
  169. build-py-38-common:
  170. steps:
  171. - checkout
  172. - run:
  173. name: compile
  174. command: |
  175. if [ -d /usr/share/hue ]; then
  176. # Running in gethue/hue Docker container
  177. apt-get update
  178. apt-get install -y python3.8-dev python3.8-venv python3.8-distutils libsnappy-dev # This should not be needed as some point
  179. curl -sL https://bootstrap.pypa.io/get-pip.py | python3.8
  180. else
  181. sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime
  182. export DEBIAN_FRONTEND=noninteractive
  183. sudo apt-get update
  184. sudo apt-get install -y gcc g++ build-essential python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev
  185. curl -sL https://deb.nodesource.com/setup_14.x | sudo bash - && sudo apt-get install -y nodejs
  186. curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python3.8
  187. sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
  188. fi
  189. export PYTHON_VER=python3.8
  190. export ROOT=$PWD
  191. make apps
  192. - run:
  193. name: run python lints
  194. command: |
  195. ./build/env/bin/pip install pylint==2.5.3 pylint-django==2.3.0 configparser==5.3.0
  196. ./tools/ci/check_for_python_lint.sh
  197. - run:
  198. name: run tests
  199. command: |
  200. PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
  201. jobs:
  202. build:
  203. docker:
  204. # specify the version you desire here
  205. # - image: circleci/python:3.8.0
  206. - image: gethue/hue:latest-py2
  207. # Specify service dependencies here if necessary
  208. # CircleCI maintains a library of pre-built images
  209. # documented at https://circleci.com/docs/2.0/circleci-images/
  210. # - image: circleci/postgres:9.4
  211. working_directory: ~/repo
  212. steps:
  213. - build-common:
  214. go_arch: "amd64"
  215. hugo_arch: "64bit"
  216. build-arm64:
  217. machine:
  218. image: ubuntu-2004:202101-01
  219. resource_class: arm.medium
  220. working_directory: ~/repo
  221. steps:
  222. - build-common:
  223. go_arch: "arm64"
  224. hugo_arch: "ARM64"
  225. commit:
  226. docker:
  227. - image: circleci/python:3.8.0
  228. working_directory: ~/repo
  229. steps:
  230. - checkout
  231. - add_ssh_keys:
  232. fingerprints:
  233. - "8c:68:29:28:04:80:38:31:c0:59:d9:3d:65:3e:b7:8c"
  234. - run:
  235. name: push to master
  236. command: |
  237. git push origin HEAD:master
  238. build-py-38:
  239. docker:
  240. - image: gethue/hue:latest-py2 # Should be circleci/python:3.6 at some point
  241. working_directory: ~/repo
  242. steps:
  243. - build-py-38-common
  244. build-py-38-arm64:
  245. machine:
  246. image: ubuntu-2004:202101-01
  247. resource_class: arm.medium
  248. working_directory: ~/repo
  249. steps:
  250. - build-py-38-common
  251. workflows:
  252. version: 2
  253. run-tests-and-rebase-master:
  254. jobs:
  255. - build:
  256. filters:
  257. branches:
  258. ignore:
  259. - master
  260. - py3-ci
  261. - build-arm64:
  262. filters:
  263. branches:
  264. ignore:
  265. - master
  266. - py3-ci
  267. - commit:
  268. requires:
  269. - build
  270. filters:
  271. branches:
  272. only:
  273. - /.*ci-commit-master.*/
  274. build-python3:
  275. jobs:
  276. - build-py-38:
  277. filters:
  278. branches:
  279. ignore:
  280. - master
  281. - build-py-38-arm64:
  282. filters:
  283. branches:
  284. ignore:
  285. - master