config.yml 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305
  1. # Python CircleCI 2.0 configuration file
  2. #
  3. # Check https://circleci.com/docs/2.0/language-python/ for more details
  4. #
  5. version: 2.1
  6. commands:
  7. build-common:
  8. parameters:
  9. go_arch:
  10. # amd64 or arm64
  11. type: string
  12. hugo_arch:
  13. # 64bit (AMD64) or ARM64
  14. type: string
  15. steps:
  16. - checkout
  17. # Download and cache dependencies
  18. # - restore_cache:
  19. # keys:
  20. # - v1-dependencies-{{ checksum "requirements.txt" }}
  21. # # fallback to using the latest cache if no exact match is found
  22. # - v1-dependencies-
  23. # - run:
  24. # name: install dependencies
  25. # command: |
  26. # python -m venv venv
  27. # . venv/bin/activate
  28. # pip install -r requirements.txt
  29. # - save_cache:
  30. # paths:
  31. # - ./venv
  32. # key: v1-dependencies-{{ checksum "requirements.txt" }}
  33. # Revert to traditional pip install when build refactored, in the meantime:
  34. - run:
  35. name: refresh sources
  36. command: |
  37. rm -r /usr/share/hue/desktop/core/src/desktop
  38. cp -r desktop/core/src/desktop /usr/share/hue/desktop/core/src/desktop
  39. for lib in `ls desktop/libs`
  40. do
  41. rm -r /usr/share/hue/desktop/libs/$lib/src/$lib
  42. cp -r desktop/libs/$lib/src/$lib /usr/share/hue/desktop/libs/$lib/src/$lib
  43. done
  44. for lib in `ls apps | grep -v Makefile`
  45. do
  46. rm -r /usr/share/hue/apps/$lib/src/$lib
  47. cp -r apps/$lib/src/$lib /usr/share/hue/apps/$lib/src/$lib
  48. done
  49. rm -r /usr/share/hue/tools
  50. cp -r tools /usr/share/hue
  51. rm /usr/share/hue/desktop/conf/*.ini
  52. cp desktop/conf/pseudo-distributed.ini.tmpl /usr/share/hue/desktop/conf/pseudo-distributed.ini
  53. # ini configuration tweaks
  54. ## Very slow if on, cuts time in two and does not skip tests
  55. sed -i 's/## has_iam_detection=true/ has_iam_detection=false/g' /usr/share/hue/desktop/conf/pseudo-distributed.ini
  56. cd /usr/share/hue
  57. # make npm-install # Not available
  58. cp ~/repo/babel.config.js .
  59. cp ~/repo/tsconfig.json .
  60. cp ~/repo/jest.config.js .
  61. cp ~/repo/.pylintrc .
  62. cp ~/repo/.stylelintrc .
  63. cp ~/repo/webpack.config*.js .
  64. rm package.json
  65. cp ~/repo/package.json .
  66. rm package-lock.json
  67. cp ~/repo/package-lock.json .
  68. npm install
  69. npm i eslint-plugin-jest@latest --save-dev # Seems to not be found otherwise
  70. npm run webpack
  71. npm run webpack-login
  72. npm run webpack-workers
  73. ./build/env/bin/hue collectstatic --noinput
  74. cp -r ~/repo/docs .
  75. # Run documentation lint
  76. - run:
  77. name: run documentation lints
  78. command: |
  79. cd ~/repo
  80. # Installs to move to image building
  81. curl -O https://dl.google.com/go/go1.15.linux-<< parameters.go_arch >>.tar.gz
  82. tar -xvf go1.15.linux-<< parameters.go_arch >>.tar.gz
  83. export GO111MODULE=on
  84. go/bin/go get -u github.com/raviqqe/muffet@v1.5.7
  85. curl --output hugo_0.69.0_Linux.tar.gz -L https://github.com/gohugoio/hugo/releases/download/v0.69.0/hugo_0.69.0_Linux-<< parameters.hugo_arch >>.tar.gz
  86. tar -xvf hugo_0.69.0_Linux.tar.gz
  87. export PATH=$PWD:$HOME/go/bin:$PATH
  88. # Trigger linting if documentation changes
  89. ./tools/ci/check_for_website_dead_links.sh docs/docs-site
  90. # ./tools/ci/check_for_website_dead_links.sh docs/gethue
  91. # Run code lints
  92. - run:
  93. name: run python lints
  94. command: |
  95. cd ~/repo
  96. /usr/share/hue/build/env/bin/pip install pylint==1.7.5 pylint-django==0.7.2 configparser==4.0.2
  97. ./tools/ci/check_for_python_lint.sh /usr/share/hue
  98. - run:
  99. name: run commit title format check
  100. command: |
  101. cd ~/repo
  102. ./tools/ci/check_for_commit_message.sh
  103. - run:
  104. name: run js lint
  105. command: |
  106. cd /usr/share/hue
  107. cp ~/repo/.prettierrc .
  108. cp ~/repo/.eslint* .
  109. cp ~/repo/tools . -r
  110. npm run lint
  111. - run:
  112. name: run style lint
  113. command: |
  114. cd /usr/share/hue
  115. npm run style-lint
  116. - run:
  117. name: run npm license checker
  118. command: |
  119. cd /usr/share/hue
  120. npm run check-license
  121. - run:
  122. name: run npm absolute path detection
  123. command: |
  124. cd /usr/share/hue
  125. npm run check-absolute-paths
  126. # Run tests
  127. - run:
  128. name: run python API tests
  129. command: |
  130. cd /usr/share/hue
  131. PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
  132. - run:
  133. name: run js tests
  134. command: |
  135. cd /usr/share/hue
  136. # https://jestjs.io/docs/en/troubleshooting.html#tests-are-extremely-slow-on-docker-andor-continuous-integration-ci-server
  137. sed -i 's/"test": "jest"/"test": "jest --runInBand"/g' package.json
  138. npm run test
  139. - store_artifacts:
  140. path: test-reports
  141. destination: test-reports
  142. build-py-36-common:
  143. steps:
  144. - checkout
  145. # Download and cache dependencies
  146. # - restore_cache:
  147. # keys:
  148. # - v1-dependencies-{{ checksum "desktop/core/requirements.txt" }}
  149. # # fallback to using the latest cache if no exact match is found
  150. # - v1-dependencies-
  151. - run:
  152. name: compile
  153. command: |
  154. apt-get update && apt-get install -y python3.6-dev libsnappy-dev asciidoc # This should not be needed as some point
  155. export PYTHON_VER=python3.6
  156. make apps
  157. make prod
  158. - run:
  159. name: run tests
  160. command: |
  161. PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
  162. # - run:
  163. # name: install dependencies
  164. # command: |
  165. # python -m venv venv
  166. # make apps
  167. # - save_cache:
  168. # paths:
  169. # - ./build/venv
  170. # key: v1-dependencies-{{ checksum "esktop/core/requirements.txt" }}
  171. build-py-38-common:
  172. steps:
  173. - checkout
  174. - run:
  175. name: compile
  176. command: |
  177. apt-get install -y python3.8-dev python3.8-venv python3.8-distutils libsnappy-dev # This should not be needed as some point
  178. export PYTHON_VER=python3.8
  179. make apps
  180. - run:
  181. name: run tests
  182. command: |
  183. PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
  184. jobs:
  185. build:
  186. docker:
  187. # specify the version you desire here
  188. # - image: circleci/python:3.8.0
  189. - image: gethue/hue:latest-py2
  190. # Specify service dependencies here if necessary
  191. # CircleCI maintains a library of pre-built images
  192. # documented at https://circleci.com/docs/2.0/circleci-images/
  193. # - image: circleci/postgres:9.4
  194. working_directory: ~/repo
  195. steps:
  196. - build-common:
  197. go_arch: "amd64"
  198. hugo_arch: "64bit"
  199. commit:
  200. docker:
  201. - image: circleci/python:3.8.0
  202. working_directory: ~/repo
  203. steps:
  204. - checkout
  205. - add_ssh_keys:
  206. fingerprints:
  207. - "8c:68:29:28:04:80:38:31:c0:59:d9:3d:65:3e:b7:8c"
  208. - run:
  209. name: push to master
  210. command: |
  211. git push origin HEAD:master
  212. build-py-36:
  213. docker:
  214. - image: gethue/hue:latest-py2 # Should be circleci/python:3.6 at some point
  215. working_directory: ~/repo
  216. steps:
  217. - build-py-36-common
  218. build-py-38:
  219. docker:
  220. - image: gethue/hue:latest-py2 # Should be circleci/python:3.6 at some point
  221. working_directory: ~/repo
  222. steps:
  223. - build-py-38-common
  224. workflows:
  225. version: 2
  226. run-tests-and-rebase-master:
  227. jobs:
  228. - build:
  229. filters:
  230. branches:
  231. ignore:
  232. - master
  233. - py3-ci
  234. - commit:
  235. requires:
  236. - build
  237. filters:
  238. branches:
  239. only:
  240. - /.*ci-commit-master.*/