config.yml 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303
  1. # Python CircleCI 2.0 configuration file
  2. #
  3. # Check https://circleci.com/docs/2.0/language-python/ for more details
  4. #
  5. version: 2
  6. jobs:
  7. build:
  8. docker:
  9. # specify the version you desire here
  10. # - image: circleci/python:3.8.0
  11. - image: gethue/hue:latest-py2
  12. # Specify service dependencies here if necessary
  13. # CircleCI maintains a library of pre-built images
  14. # documented at https://circleci.com/docs/2.0/circleci-images/
  15. # - image: circleci/postgres:9.4
  16. working_directory: ~/repo
  17. steps:
  18. - checkout
  19. # Download and cache dependencies
  20. # - restore_cache:
  21. # keys:
  22. # - v1-dependencies-{{ checksum "requirements.txt" }}
  23. # # fallback to using the latest cache if no exact match is found
  24. # - v1-dependencies-
  25. # - run:
  26. # name: install dependencies
  27. # command: |
  28. # python -m venv venv
  29. # . venv/bin/activate
  30. # pip install -r requirements.txt
  31. # - save_cache:
  32. # paths:
  33. # - ./venv
  34. # key: v1-dependencies-{{ checksum "requirements.txt" }}
  35. # Revert to traditional pip install when build refactored, in the meantime:
  36. - run:
  37. name: refresh sources
  38. command: |
  39. rm -r /usr/share/hue/desktop/core/src/desktop
  40. cp -r desktop/core/src/desktop /usr/share/hue/desktop/core/src/desktop
  41. for lib in `ls desktop/libs`
  42. do
  43. rm -r /usr/share/hue/desktop/libs/$lib/src/$lib
  44. cp -r desktop/libs/$lib/src/$lib /usr/share/hue/desktop/libs/$lib/src/$lib
  45. done
  46. for lib in `ls apps | grep -v Makefile`
  47. do
  48. rm -r /usr/share/hue/apps/$lib/src/$lib
  49. cp -r apps/$lib/src/$lib /usr/share/hue/apps/$lib/src/$lib
  50. done
  51. rm -r /usr/share/hue/tools
  52. cp -r tools /usr/share/hue
  53. rm /usr/share/hue/desktop/conf/*.ini
  54. cp desktop/conf/pseudo-distributed.ini.tmpl /usr/share/hue/desktop/conf/pseudo-distributed.ini
  55. # ini configuration tweaks
  56. ## Very slow if on, cuts time in two and does not skip tests
  57. sed -i 's/## has_iam_detection=true/ has_iam_detection=false/g' /usr/share/hue/desktop/conf/pseudo-distributed.ini
  58. cd /usr/share/hue
  59. # make npm-install # Not available
  60. cp ~/repo/babel.config.js .
  61. cp ~/repo/tsconfig.json .
  62. cp ~/repo/jest.config.js .
  63. cp ~/repo/.pylintrc .
  64. cp ~/repo/.stylelintrc .
  65. cp ~/repo/webpack.config*.js .
  66. rm package.json
  67. cp ~/repo/package.json .
  68. rm package-lock.json
  69. cp ~/repo/package-lock.json .
  70. npm install
  71. npm i eslint-plugin-jest@latest --save-dev # Seems to not be found otherwise
  72. npm run webpack
  73. npm run webpack-login
  74. npm run webpack-workers
  75. ./build/env/bin/hue collectstatic --noinput
  76. cp -r ~/repo/docs .
  77. # Run documentation lint
  78. - run:
  79. name: run documentation lints
  80. command: |
  81. cd ~/repo
  82. # Installs to move to image building
  83. curl -O https://dl.google.com/go/go1.15.linux-amd64.tar.gz
  84. tar -xvf go1.15.linux-amd64.tar.gz
  85. export GO111MODULE=on
  86. go/bin/go get -u github.com/raviqqe/muffet@v1.5.7
  87. curl --output hugo_0.69.0_Linux-64bit.tar.gz -L https://github.com/gohugoio/hugo/releases/download/v0.69.0/hugo_0.69.0_Linux-64bit.tar.gz
  88. tar -xvf hugo_0.69.0_Linux-64bit.tar.gz
  89. export PATH=$PWD:$HOME/go/bin:$PATH
  90. # Trigger linting if documentation changes
  91. ./tools/ci/check_for_website_dead_links.sh docs/docs-site
  92. # ./tools/ci/check_for_website_dead_links.sh docs/gethue
  93. # Run code lints
  94. - run:
  95. name: run python lints
  96. command: |
  97. cd ~/repo
  98. /usr/share/hue/build/env/bin/pip install pylint==1.7.5 pylint-django==0.7.2 configparser==4.0.2
  99. ./tools/ci/check_for_python_lint.sh /usr/share/hue
  100. - run:
  101. name: run commit title format check
  102. command: |
  103. cd ~/repo
  104. ./tools/ci/check_for_commit_message.sh
  105. - run:
  106. name: run js lint
  107. command: |
  108. cd /usr/share/hue
  109. cp ~/repo/.prettierrc .
  110. cp ~/repo/.eslint* .
  111. cp ~/repo/tools . -r
  112. npm run lint
  113. - run:
  114. name: run style lint
  115. command: |
  116. cd /usr/share/hue
  117. npm run style-lint
  118. - run:
  119. name: run npm license checker
  120. command: |
  121. cd /usr/share/hue
  122. npm run check-license
  123. - run:
  124. name: run npm absolute path detection
  125. command: |
  126. cd /usr/share/hue
  127. npm run check-absolute-paths
  128. # Run tests
  129. - run:
  130. name: run python API tests
  131. command: |
  132. cd /usr/share/hue
  133. PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
  134. - run:
  135. name: run js tests
  136. command: |
  137. cd /usr/share/hue
  138. # https://jestjs.io/docs/en/troubleshooting.html#tests-are-extremely-slow-on-docker-andor-continuous-integration-ci-server
  139. sed -i 's/"test": "jest"/"test": "jest --runInBand"/g' package.json
  140. npm run test
  141. - store_artifacts:
  142. path: test-reports
  143. destination: test-reports
  144. commit:
  145. docker:
  146. - image: circleci/python:3.8.0
  147. working_directory: ~/repo
  148. steps:
  149. - checkout
  150. - add_ssh_keys:
  151. fingerprints:
  152. - "8c:68:29:28:04:80:38:31:c0:59:d9:3d:65:3e:b7:8c"
  153. - run:
  154. name: push to master
  155. command: |
  156. git push origin HEAD:master
  157. build-py3.6:
  158. docker:
  159. - image: gethue/hue:latest-py2 # Should be circleci/python:3.6 at some point
  160. working_directory: ~/repo
  161. steps:
  162. - checkout
  163. # Download and cache dependencies
  164. # - restore_cache:
  165. # keys:
  166. # - v1-dependencies-{{ checksum "desktop/core/requirements.txt" }}
  167. # # fallback to using the latest cache if no exact match is found
  168. # - v1-dependencies-
  169. - run:
  170. name: compile
  171. command: |
  172. apt-get update && apt-get install -y python3.6-dev libsnappy-dev asciidoc # This should not be needed as some point
  173. export PYTHON_VER=python3.6
  174. make apps
  175. make prod
  176. - run:
  177. name: run tests
  178. command: |
  179. PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
  180. # - run:
  181. # name: install dependencies
  182. # command: |
  183. # python -m venv venv
  184. # make apps
  185. # - save_cache:
  186. # paths:
  187. # - ./build/venv
  188. # key: v1-dependencies-{{ checksum "esktop/core/requirements.txt" }}
  189. build-py3.8:
  190. docker:
  191. - image: gethue/hue:latest-py2 # Should be circleci/python:3.x at some point
  192. working_directory: ~/repo
  193. steps:
  194. - checkout
  195. - run:
  196. name: compile
  197. command: |
  198. apt-get install -y python3.8-dev python3.8-venv python3.8-distutils libsnappy-dev # This should not be needed as some point
  199. export PYTHON_VER=python3.8
  200. make apps
  201. - run:
  202. name: run tests
  203. command: |
  204. PYTHONWARNINGS=always ./build/env/bin/hue test unit --with-xunit --with-cover
  205. workflows:
  206. version: 2
  207. run-tests-and-rebase-master:
  208. jobs:
  209. - build:
  210. filters:
  211. branches:
  212. ignore:
  213. - master
  214. - py3-ci
  215. - build-py3.6:
  216. filters:
  217. branches:
  218. ignore:
  219. - master
  220. - py3-ci
  221. - commit:
  222. requires:
  223. - build
  224. - build-py3.6
  225. filters:
  226. branches:
  227. only:
  228. - /.*ci-commit-master.*/
  229. build-python3:
  230. jobs:
  231. - build-py3.8:
  232. filters:
  233. branches:
  234. only:
  235. - py3-ci
  236. - build-py3.6:
  237. filters:
  238. branches:
  239. only:
  240. - py3-ci