setupinfo.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563
  1. import sys
  2. import io
  3. import os
  4. import os.path
  5. import subprocess
  6. from setuptools.command.build_ext import build_ext as _build_ext
  7. from distutils.core import Extension
  8. from distutils.errors import CompileError, DistutilsOptionError
  9. from versioninfo import get_base_dir
  10. try:
  11. import Cython.Compiler.Version
  12. CYTHON_INSTALLED = True
  13. except ImportError:
  14. CYTHON_INSTALLED = False
  15. EXT_MODULES = ["lxml.etree", "lxml.objectify"]
  16. COMPILED_MODULES = [
  17. "lxml.builder",
  18. "lxml._elementpath",
  19. "lxml.html.diff",
  20. "lxml.html.clean",
  21. "lxml.sax",
  22. ]
  23. HEADER_FILES = ['etree.h', 'etree_api.h']
  24. if hasattr(sys, 'pypy_version_info') or (
  25. getattr(sys, 'implementation', None) and sys.implementation.name != 'cpython'):
  26. # disable Cython compilation of Python modules in PyPy and other non-CPythons
  27. del COMPILED_MODULES[:]
  28. SOURCE_PATH = "src"
  29. INCLUDE_PACKAGE_PATH = os.path.join(SOURCE_PATH, 'lxml', 'includes')
  30. if sys.version_info[0] >= 3:
  31. _system_encoding = sys.getdefaultencoding()
  32. if _system_encoding is None:
  33. _system_encoding = "iso-8859-1" # :-)
  34. def decode_input(data):
  35. if isinstance(data, str):
  36. return data
  37. return data.decode(_system_encoding)
  38. else:
  39. def decode_input(data):
  40. return data
  41. def env_var(name):
  42. value = os.getenv(name)
  43. if value:
  44. value = decode_input(value)
  45. if sys.platform == 'win32' and ';' in value:
  46. return value.split(';')
  47. else:
  48. return value.split()
  49. else:
  50. return []
  51. def _prefer_reldirs(base_dir, dirs):
  52. return [
  53. os.path.relpath(path) if path.startswith(base_dir) else path
  54. for path in dirs
  55. ]
  56. def ext_modules(static_include_dirs, static_library_dirs,
  57. static_cflags, static_binaries):
  58. global XML2_CONFIG, XSLT_CONFIG
  59. if OPTION_BUILD_LIBXML2XSLT:
  60. from buildlibxml import build_libxml2xslt, get_prebuilt_libxml2xslt
  61. if sys.platform.startswith('win'):
  62. get_prebuilt_libxml2xslt(
  63. OPTION_DOWNLOAD_DIR, static_include_dirs, static_library_dirs)
  64. else:
  65. XML2_CONFIG, XSLT_CONFIG = build_libxml2xslt(
  66. OPTION_DOWNLOAD_DIR, 'build/tmp',
  67. static_include_dirs, static_library_dirs,
  68. static_cflags, static_binaries,
  69. libiconv_version=OPTION_LIBICONV_VERSION,
  70. libxml2_version=OPTION_LIBXML2_VERSION,
  71. libxslt_version=OPTION_LIBXSLT_VERSION,
  72. zlib_version=OPTION_ZLIB_VERSION,
  73. multicore=OPTION_MULTICORE)
  74. modules = EXT_MODULES + COMPILED_MODULES
  75. if OPTION_WITHOUT_OBJECTIFY:
  76. modules = [entry for entry in modules if 'objectify' not in entry]
  77. module_files = list(os.path.join(SOURCE_PATH, *module.split('.')) for module in modules)
  78. c_files_exist = [os.path.exists(module + '.c') for module in module_files]
  79. use_cython = True
  80. if CYTHON_INSTALLED and (OPTION_WITH_CYTHON or not all(c_files_exist)):
  81. print("Building with Cython %s." % Cython.Compiler.Version.version)
  82. # generate module cleanup code
  83. from Cython.Compiler import Options
  84. Options.generate_cleanup_code = 3
  85. Options.clear_to_none = False
  86. elif not OPTION_WITHOUT_CYTHON and not all(c_files_exist):
  87. for exists, module in zip(c_files_exist, module_files):
  88. if not exists:
  89. raise RuntimeError(
  90. "ERROR: Trying to build without Cython, but pre-generated '%s.c' "
  91. "is not available (pass --without-cython to ignore this error)." % module)
  92. else:
  93. if not all(c_files_exist):
  94. for exists, module in zip(c_files_exist, module_files):
  95. if not exists:
  96. print("WARNING: Trying to build without Cython, but pre-generated "
  97. "'%s.c' is not available." % module)
  98. use_cython = False
  99. print("Building without Cython.")
  100. if not check_build_dependencies():
  101. raise RuntimeError("Dependency missing")
  102. base_dir = get_base_dir()
  103. _include_dirs = _prefer_reldirs(
  104. base_dir, include_dirs(static_include_dirs) + [
  105. SOURCE_PATH,
  106. INCLUDE_PACKAGE_PATH,
  107. ])
  108. _library_dirs = _prefer_reldirs(base_dir, library_dirs(static_library_dirs))
  109. _cflags = cflags(static_cflags)
  110. _ldflags = ['-isysroot', get_xcode_isysroot()] if sys.platform == 'darwin' else None
  111. _define_macros = define_macros()
  112. _libraries = libraries()
  113. if _library_dirs:
  114. message = "Building against libxml2/libxslt in "
  115. if len(_library_dirs) > 1:
  116. print(message + "one of the following directories:")
  117. for dir in _library_dirs:
  118. print(" " + dir)
  119. else:
  120. print(message + "the following directory: " +
  121. _library_dirs[0])
  122. if OPTION_AUTO_RPATH:
  123. runtime_library_dirs = _library_dirs
  124. else:
  125. runtime_library_dirs = []
  126. if CYTHON_INSTALLED and OPTION_SHOW_WARNINGS:
  127. from Cython.Compiler import Errors
  128. Errors.LEVEL = 0
  129. cythonize_directives = {
  130. 'binding': True,
  131. }
  132. if OPTION_WITH_COVERAGE:
  133. cythonize_directives['linetrace'] = True
  134. result = []
  135. for module, src_file in zip(modules, module_files):
  136. is_py = module in COMPILED_MODULES
  137. main_module_source = src_file + (
  138. '.c' if not use_cython else '.py' if is_py else '.pyx')
  139. result.append(
  140. Extension(
  141. module,
  142. sources = [main_module_source],
  143. depends = find_dependencies(module),
  144. extra_compile_args = _cflags,
  145. extra_link_args = None if is_py else _ldflags,
  146. extra_objects = None if is_py else static_binaries,
  147. define_macros = _define_macros,
  148. include_dirs = _include_dirs,
  149. library_dirs = None if is_py else _library_dirs,
  150. runtime_library_dirs = None if is_py else runtime_library_dirs,
  151. libraries = None if is_py else _libraries,
  152. ))
  153. if CYTHON_INSTALLED and OPTION_WITH_CYTHON_GDB:
  154. for ext in result:
  155. ext.cython_gdb = True
  156. if CYTHON_INSTALLED and use_cython:
  157. # build .c files right now and convert Extension() objects
  158. from Cython.Build import cythonize
  159. result = cythonize(result, compiler_directives=cythonize_directives)
  160. # for backwards compatibility reasons, provide "etree[_api].h" also as "lxml.etree[_api].h"
  161. for header_filename in HEADER_FILES:
  162. src_file = os.path.join(SOURCE_PATH, 'lxml', header_filename)
  163. dst_file = os.path.join(SOURCE_PATH, 'lxml', 'lxml.' + header_filename)
  164. if not os.path.exists(src_file):
  165. continue
  166. if os.path.exists(dst_file) and os.path.getmtime(dst_file) >= os.path.getmtime(src_file):
  167. continue
  168. with io.open(src_file, 'r', encoding='iso8859-1') as f:
  169. content = f.read()
  170. for filename in HEADER_FILES:
  171. content = content.replace('"%s"' % filename, '"lxml.%s"' % filename)
  172. with io.open(dst_file, 'w', encoding='iso8859-1') as f:
  173. f.write(content)
  174. return result
  175. def find_dependencies(module):
  176. if not CYTHON_INSTALLED or 'lxml.html' in module:
  177. return []
  178. base_dir = get_base_dir()
  179. package_dir = os.path.join(base_dir, SOURCE_PATH, 'lxml')
  180. includes_dir = os.path.join(base_dir, INCLUDE_PACKAGE_PATH)
  181. pxd_files = [
  182. os.path.join(INCLUDE_PACKAGE_PATH, filename)
  183. for filename in os.listdir(includes_dir)
  184. if filename.endswith('.pxd')
  185. ]
  186. if module == 'lxml.etree':
  187. pxi_files = [
  188. os.path.join(SOURCE_PATH, 'lxml', filename)
  189. for filename in os.listdir(package_dir)
  190. if filename.endswith('.pxi') and 'objectpath' not in filename
  191. ]
  192. pxd_files = [
  193. filename for filename in pxd_files
  194. if 'etreepublic' not in filename
  195. ]
  196. elif module == 'lxml.objectify':
  197. pxi_files = [os.path.join(SOURCE_PATH, 'lxml', 'objectpath.pxi')]
  198. else:
  199. pxi_files = pxd_files = []
  200. return pxd_files + pxi_files
  201. def extra_setup_args():
  202. class CheckLibxml2BuildExt(_build_ext):
  203. """Subclass to check whether libxml2 is really available if the build fails"""
  204. def run(self):
  205. try:
  206. _build_ext.run(self) # old-style class in Py2
  207. except CompileError as e:
  208. print('Compile failed: %s' % e)
  209. if not seems_to_have_libxml2():
  210. print_libxml_error()
  211. raise
  212. result = {'cmdclass': {'build_ext': CheckLibxml2BuildExt}}
  213. return result
  214. def seems_to_have_libxml2():
  215. from distutils import ccompiler
  216. compiler = ccompiler.new_compiler()
  217. return compiler.has_function(
  218. 'xmlXPathInit',
  219. include_dirs=include_dirs([]) + ['/usr/include/libxml2'],
  220. includes=['libxml/xpath.h'],
  221. library_dirs=library_dirs([]),
  222. libraries=['xml2'])
  223. def print_libxml_error():
  224. print('*********************************************************************************')
  225. print('Could not find function xmlCheckVersion in library libxml2. Is libxml2 installed?')
  226. if sys.platform in ('darwin',):
  227. print('Perhaps try: xcode-select --install')
  228. print('*********************************************************************************')
  229. def libraries():
  230. standard_libs = []
  231. if 'linux' in sys.platform:
  232. standard_libs.append('rt')
  233. if not OPTION_BUILD_LIBXML2XSLT:
  234. standard_libs.append('z')
  235. standard_libs.append('m')
  236. if sys.platform in ('win32',):
  237. libs = ['libxslt', 'libexslt', 'libxml2', 'iconv']
  238. if OPTION_STATIC:
  239. libs = ['%s_a' % lib for lib in libs]
  240. libs.extend(['zlib', 'WS2_32'])
  241. elif OPTION_STATIC:
  242. libs = standard_libs
  243. else:
  244. libs = ['xslt', 'exslt', 'xml2'] + standard_libs
  245. return libs
  246. def library_dirs(static_library_dirs):
  247. if OPTION_STATIC:
  248. if not static_library_dirs:
  249. static_library_dirs = env_var('LIBRARY')
  250. assert static_library_dirs, "Static build not configured, see doc/build.txt"
  251. return static_library_dirs
  252. # filter them from xslt-config --libs
  253. result = []
  254. possible_library_dirs = flags('libs')
  255. for possible_library_dir in possible_library_dirs:
  256. if possible_library_dir.startswith('-L'):
  257. result.append(possible_library_dir[2:])
  258. return result
  259. def include_dirs(static_include_dirs):
  260. if OPTION_STATIC:
  261. if not static_include_dirs:
  262. static_include_dirs = env_var('INCLUDE')
  263. return static_include_dirs
  264. # filter them from xslt-config --cflags
  265. result = []
  266. possible_include_dirs = flags('cflags')
  267. for possible_include_dir in possible_include_dirs:
  268. if possible_include_dir.startswith('-I'):
  269. result.append(possible_include_dir[2:])
  270. return result
  271. def cflags(static_cflags):
  272. result = []
  273. if not OPTION_SHOW_WARNINGS:
  274. result.append('-w')
  275. if OPTION_DEBUG_GCC:
  276. result.append('-g2')
  277. if OPTION_STATIC:
  278. if not static_cflags:
  279. static_cflags = env_var('CFLAGS')
  280. result.extend(static_cflags)
  281. else:
  282. # anything from xslt-config --cflags that doesn't start with -I
  283. possible_cflags = flags('cflags')
  284. for possible_cflag in possible_cflags:
  285. if not possible_cflag.startswith('-I'):
  286. result.append(possible_cflag)
  287. if sys.platform in ('darwin',):
  288. for opt in result:
  289. if 'flat_namespace' in opt:
  290. break
  291. else:
  292. result.append('-flat_namespace')
  293. return result
  294. def define_macros():
  295. macros = []
  296. if OPTION_WITHOUT_ASSERT:
  297. macros.append(('PYREX_WITHOUT_ASSERTIONS', None))
  298. if OPTION_WITHOUT_THREADING:
  299. macros.append(('WITHOUT_THREADING', None))
  300. if OPTION_WITH_REFNANNY:
  301. macros.append(('CYTHON_REFNANNY', None))
  302. if OPTION_WITH_UNICODE_STRINGS:
  303. macros.append(('LXML_UNICODE_STRINGS', '1'))
  304. if OPTION_WITH_COVERAGE:
  305. macros.append(('CYTHON_TRACE_NOGIL', '1'))
  306. if OPTION_BUILD_LIBXML2XSLT:
  307. macros.append(('LIBXML_STATIC', None))
  308. macros.append(('LIBXSLT_STATIC', None))
  309. # Disable showing C lines in tracebacks, unless explicitly requested.
  310. macros.append(('CYTHON_CLINE_IN_TRACEBACK', '1' if OPTION_WITH_CLINES else '0'))
  311. return macros
  312. def run_command(cmd, *args):
  313. if not cmd:
  314. return ''
  315. if args:
  316. cmd = ' '.join((cmd,) + args)
  317. p = subprocess.Popen(cmd, shell=True,
  318. stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  319. stdout_data, errors = p.communicate()
  320. if p.returncode != 0 and errors:
  321. return ''
  322. return decode_input(stdout_data).strip()
  323. def check_min_version(version, min_version, libname):
  324. if not version:
  325. # this is ok for targets like sdist etc.
  326. return True
  327. lib_version = tuple(map(int, version.split('.')[:3]))
  328. req_version = tuple(map(int, min_version.split('.')[:3]))
  329. if lib_version < req_version:
  330. print("Minimum required version of %s is %s. Your system has version %s." % (
  331. libname, min_version, version))
  332. return False
  333. return True
  334. def get_library_version(prog, libname=None):
  335. if libname:
  336. return run_command(prog, '--modversion %s' % libname)
  337. else:
  338. return run_command(prog, '--version')
  339. PKG_CONFIG = None
  340. XML2_CONFIG = None
  341. XSLT_CONFIG = None
  342. def get_library_versions():
  343. global XML2_CONFIG, XSLT_CONFIG
  344. # Pre-built libraries
  345. if XML2_CONFIG and XSLT_CONFIG:
  346. xml2_version = get_library_version(XML2_CONFIG)
  347. xslt_version = get_library_version(XSLT_CONFIG)
  348. return xml2_version, xslt_version
  349. # Path to xml2-config and xslt-config specified on the command line
  350. if OPTION_WITH_XML2_CONFIG:
  351. xml2_version = get_library_version(OPTION_WITH_XML2_CONFIG)
  352. if xml2_version and OPTION_WITH_XSLT_CONFIG:
  353. xslt_version = get_library_version(OPTION_WITH_XSLT_CONFIG)
  354. if xslt_version:
  355. XML2_CONFIG = OPTION_WITH_XML2_CONFIG
  356. XSLT_CONFIG = OPTION_WITH_XSLT_CONFIG
  357. return xml2_version, xslt_version
  358. # Try pkg-config
  359. global PKG_CONFIG
  360. PKG_CONFIG = os.getenv('PKG_CONFIG', 'pkg-config')
  361. xml2_version = get_library_version(PKG_CONFIG, 'libxml-2.0')
  362. if xml2_version:
  363. xslt_version = get_library_version(PKG_CONFIG, 'libxslt')
  364. if xml2_version and xslt_version:
  365. return xml2_version, xslt_version
  366. # Try xml2-config and xslt-config
  367. XML2_CONFIG = os.getenv('XML2_CONFIG', 'xml2-config')
  368. xml2_version = get_library_version(XML2_CONFIG)
  369. if xml2_version:
  370. XSLT_CONFIG = os.getenv('XSLT_CONFIG', 'xslt-config')
  371. xslt_version = get_library_version(XSLT_CONFIG)
  372. if xml2_version and xslt_version:
  373. return xml2_version, xslt_version
  374. # One or both build dependencies not found. Fail on Linux platforms only.
  375. if sys.platform.startswith('win'):
  376. return '', ''
  377. print("Error: Please make sure the libxml2 and libxslt development packages are installed.")
  378. sys.exit(1)
  379. def check_build_dependencies():
  380. xml2_version, xslt_version = get_library_versions()
  381. xml2_ok = check_min_version(xml2_version, '2.7.0', 'libxml2')
  382. xslt_ok = check_min_version(xslt_version, '1.1.23', 'libxslt')
  383. if not OPTION_BUILD_LIBXML2XSLT and xml2_version in ('2.9.11', '2.9.12'):
  384. print("\n"
  385. "WARNING: The stock libxml2 versions 2.9.11 and 2.9.12 are incompatible"
  386. " with this lxml version. "
  387. "They produce excess content on serialisation. "
  388. "Use a different library version or a static build."
  389. "\n")
  390. if xml2_version and xslt_version:
  391. print("Building against libxml2 %s and libxslt %s" % (xml2_version, xslt_version))
  392. else:
  393. print("Building against pre-built libxml2 andl libxslt libraries")
  394. return (xml2_ok and xslt_ok)
  395. def get_flags(prog, option, libname=None):
  396. if libname:
  397. return run_command(prog, '--%s %s' % (option, libname))
  398. else:
  399. return run_command(prog, '--%s' % option)
  400. def flags(option):
  401. if XML2_CONFIG:
  402. xml2_flags = get_flags(XML2_CONFIG, option)
  403. xslt_flags = get_flags(XSLT_CONFIG, option)
  404. else:
  405. xml2_flags = get_flags(PKG_CONFIG, option, 'libxml-2.0')
  406. xslt_flags = get_flags(PKG_CONFIG, option, 'libxslt')
  407. flag_list = xml2_flags.split()
  408. for flag in xslt_flags.split():
  409. if flag not in flag_list:
  410. flag_list.append(flag)
  411. return flag_list
  412. def get_xcode_isysroot():
  413. return run_command('xcrun', '--show-sdk-path')
  414. ## Option handling:
  415. def has_option(name):
  416. try:
  417. sys.argv.remove('--%s' % name)
  418. return True
  419. except ValueError:
  420. pass
  421. # allow passing all cmd line options also as environment variables
  422. env_val = os.getenv(name.upper().replace('-', '_'), 'false').lower()
  423. if env_val == "true":
  424. return True
  425. return False
  426. def option_value(name, deprecated_for=None):
  427. for index, option in enumerate(sys.argv):
  428. if option == '--' + name:
  429. if index+1 >= len(sys.argv):
  430. raise DistutilsOptionError(
  431. 'The option %s requires a value' % option)
  432. value = sys.argv[index+1]
  433. sys.argv[index:index+2] = []
  434. if deprecated_for:
  435. print_deprecated_option(name, deprecated_for)
  436. return value
  437. if option.startswith('--' + name + '='):
  438. value = option[len(name)+3:]
  439. sys.argv[index:index+1] = []
  440. if deprecated_for:
  441. print_deprecated_option(name, deprecated_for)
  442. return value
  443. env_name = name.upper().replace('-', '_')
  444. env_val = os.getenv(env_name)
  445. if env_val and deprecated_for:
  446. print_deprecated_option(env_name, deprecated_for.upper().replace('-', '_'))
  447. return env_val
  448. def print_deprecated_option(name, new_name):
  449. print("WARN: Option '%s' is deprecated. Use '%s' instead." % (name, new_name))
  450. staticbuild = bool(os.environ.get('STATICBUILD', ''))
  451. # pick up any commandline options and/or env variables
  452. OPTION_WITHOUT_OBJECTIFY = has_option('without-objectify')
  453. OPTION_WITH_UNICODE_STRINGS = has_option('with-unicode-strings')
  454. OPTION_WITHOUT_ASSERT = has_option('without-assert')
  455. OPTION_WITHOUT_THREADING = has_option('without-threading')
  456. OPTION_WITHOUT_CYTHON = has_option('without-cython')
  457. OPTION_WITH_CYTHON = has_option('with-cython')
  458. OPTION_WITH_CYTHON_GDB = has_option('cython-gdb')
  459. OPTION_WITH_REFNANNY = has_option('with-refnanny')
  460. OPTION_WITH_COVERAGE = has_option('with-coverage')
  461. OPTION_WITH_CLINES = has_option('with-clines')
  462. if OPTION_WITHOUT_CYTHON:
  463. CYTHON_INSTALLED = False
  464. OPTION_STATIC = staticbuild or has_option('static')
  465. OPTION_DEBUG_GCC = has_option('debug-gcc')
  466. OPTION_SHOW_WARNINGS = has_option('warnings')
  467. OPTION_AUTO_RPATH = has_option('auto-rpath')
  468. OPTION_BUILD_LIBXML2XSLT = staticbuild or has_option('static-deps')
  469. if OPTION_BUILD_LIBXML2XSLT:
  470. OPTION_STATIC = True
  471. OPTION_WITH_XML2_CONFIG = option_value('with-xml2-config') or option_value('xml2-config', deprecated_for='with-xml2-config')
  472. OPTION_WITH_XSLT_CONFIG = option_value('with-xslt-config') or option_value('xslt-config', deprecated_for='with-xslt-config')
  473. OPTION_LIBXML2_VERSION = option_value('libxml2-version')
  474. OPTION_LIBXSLT_VERSION = option_value('libxslt-version')
  475. OPTION_LIBICONV_VERSION = option_value('libiconv-version')
  476. OPTION_ZLIB_VERSION = option_value('zlib-version')
  477. OPTION_MULTICORE = option_value('multicore')
  478. OPTION_DOWNLOAD_DIR = option_value('download-dir')
  479. if OPTION_DOWNLOAD_DIR is None:
  480. OPTION_DOWNLOAD_DIR = 'libs'