buildlibxml.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527
  1. import os, re, sys, subprocess, platform
  2. import tarfile
  3. from distutils import log, version
  4. from contextlib import closing, contextmanager
  5. from ftplib import FTP
  6. try:
  7. from urlparse import urljoin, unquote, urlparse
  8. from urllib import urlretrieve, urlopen, urlcleanup
  9. except ImportError:
  10. from urllib.parse import urljoin, unquote, urlparse
  11. from urllib.request import urlretrieve, urlopen, urlcleanup
  12. multi_make_options = []
  13. try:
  14. import multiprocessing
  15. cpus = multiprocessing.cpu_count()
  16. if cpus > 1:
  17. if cpus > 5:
  18. cpus = 5
  19. multi_make_options = ['-j%d' % (cpus+1)]
  20. except:
  21. pass
  22. # use pre-built libraries on Windows
  23. def download_and_extract_windows_binaries(destdir):
  24. url = "https://github.com/lxml/libxml2-win-binaries/releases"
  25. filenames = list(_list_dir_urllib(url))
  26. release_path = "/download/%s/" % find_max_version(
  27. "library release", filenames, re.compile(r"/releases/tag/([0-9.]+[0-9])$"))
  28. url += release_path
  29. filenames = [
  30. filename.rsplit('/', 1)[1]
  31. for filename in filenames
  32. if release_path in filename
  33. ]
  34. # Check for native ARM64 build or the environment variable that is set by
  35. # Visual Studio for cross-compilation (same variable as setuptools uses)
  36. if platform.machine() == 'ARM64' or os.getenv('VSCMD_ARG_TGT_ARCH') == 'arm64':
  37. arch = "win-arm64"
  38. elif sys.maxsize > 2**32:
  39. arch = "win64"
  40. else:
  41. arch = "win32"
  42. if sys.version_info < (3, 5):
  43. arch = 'vs2008.' + arch
  44. libs = {}
  45. for libname in ['libxml2', 'libxslt', 'zlib', 'iconv']:
  46. libs[libname] = "%s-%s.%s.zip" % (
  47. libname,
  48. find_max_version(libname, filenames),
  49. arch,
  50. )
  51. if not os.path.exists(destdir):
  52. os.makedirs(destdir)
  53. for libname, libfn in libs.items():
  54. srcfile = urljoin(url, libfn)
  55. destfile = os.path.join(destdir, libfn)
  56. if os.path.exists(destfile + ".keep"):
  57. print('Using local copy of "{}"'.format(srcfile))
  58. else:
  59. print('Retrieving "%s" to "%s"' % (srcfile, destfile))
  60. urlcleanup() # work around FTP bug 27973 in Py2.7.12+
  61. urlretrieve(srcfile, destfile)
  62. d = unpack_zipfile(destfile, destdir)
  63. libs[libname] = d
  64. return libs
  65. def find_top_dir_of_zipfile(zipfile):
  66. topdir = None
  67. files = [f.filename for f in zipfile.filelist]
  68. dirs = [d for d in files if d.endswith('/')]
  69. if dirs:
  70. dirs.sort(key=len)
  71. topdir = dirs[0]
  72. topdir = topdir[:topdir.index("/")+1]
  73. for path in files:
  74. if not path.startswith(topdir):
  75. topdir = None
  76. break
  77. assert topdir, (
  78. "cannot determine single top-level directory in zip file %s" %
  79. zipfile.filename)
  80. return topdir.rstrip('/')
  81. def unpack_zipfile(zipfn, destdir):
  82. assert zipfn.endswith('.zip')
  83. import zipfile
  84. print('Unpacking %s into %s' % (os.path.basename(zipfn), destdir))
  85. f = zipfile.ZipFile(zipfn)
  86. try:
  87. extracted_dir = os.path.join(destdir, find_top_dir_of_zipfile(f))
  88. f.extractall(path=destdir)
  89. finally:
  90. f.close()
  91. assert os.path.exists(extracted_dir), 'missing: %s' % extracted_dir
  92. return extracted_dir
  93. def get_prebuilt_libxml2xslt(download_dir, static_include_dirs, static_library_dirs):
  94. assert sys.platform.startswith('win')
  95. libs = download_and_extract_windows_binaries(download_dir)
  96. for libname, path in libs.items():
  97. i = os.path.join(path, 'include')
  98. l = os.path.join(path, 'lib')
  99. assert os.path.exists(i), 'does not exist: %s' % i
  100. assert os.path.exists(l), 'does not exist: %s' % l
  101. static_include_dirs.append(i)
  102. static_library_dirs.append(l)
  103. ## Routines to download and build libxml2/xslt from sources:
  104. LIBXML2_LOCATION = 'https://download.gnome.org/sources/libxml2/'
  105. LIBXSLT_LOCATION = 'https://download.gnome.org/sources/libxslt/'
  106. LIBICONV_LOCATION = 'https://ftp.gnu.org/pub/gnu/libiconv/'
  107. ZLIB_LOCATION = 'https://zlib.net/'
  108. match_libfile_version = re.compile('^[^-]*-([.0-9-]+)[.].*').match
  109. def _find_content_encoding(response, default='iso8859-1'):
  110. from email.message import Message
  111. content_type = response.headers.get('Content-Type')
  112. if content_type:
  113. msg = Message()
  114. msg.add_header('Content-Type', content_type)
  115. charset = msg.get_content_charset(default)
  116. else:
  117. charset = default
  118. return charset
  119. def remote_listdir(url):
  120. try:
  121. return _list_dir_urllib(url)
  122. except IOError:
  123. assert url.lower().startswith('ftp://')
  124. print("Requesting with urllib failed. Falling back to ftplib. "
  125. "Proxy argument will be ignored for %s" % url)
  126. return _list_dir_ftplib(url)
  127. def _list_dir_ftplib(url):
  128. parts = urlparse(url)
  129. ftp = FTP(parts.netloc)
  130. try:
  131. ftp.login()
  132. ftp.cwd(parts.path)
  133. data = []
  134. ftp.dir(data.append)
  135. finally:
  136. ftp.quit()
  137. return parse_text_ftplist("\n".join(data))
  138. def _list_dir_urllib(url):
  139. with closing(urlopen(url)) as res:
  140. charset = _find_content_encoding(res)
  141. content_type = res.headers.get('Content-Type')
  142. data = res.read()
  143. data = data.decode(charset)
  144. if content_type and content_type.startswith('text/html'):
  145. files = parse_html_filelist(data)
  146. else:
  147. files = parse_text_ftplist(data)
  148. return files
  149. def http_find_latest_version_directory(url):
  150. with closing(urlopen(url)) as res:
  151. charset = _find_content_encoding(res)
  152. data = res.read()
  153. # e.g. <a href="1.0/">
  154. directories = [
  155. (int(v[0]), int(v[1]))
  156. for v in re.findall(r' href=["\']([0-9]+)\.([0-9]+)/?["\']', data.decode(charset))
  157. ]
  158. if not directories:
  159. return url
  160. latest_dir = "%s.%s" % max(directories)
  161. return urljoin(url, latest_dir) + "/"
  162. def http_listfiles(url, re_pattern):
  163. with closing(urlopen(url)) as res:
  164. charset = _find_content_encoding(res)
  165. data = res.read()
  166. files = re.findall(re_pattern, data.decode(charset))
  167. return files
  168. def parse_text_ftplist(s):
  169. for line in s.splitlines():
  170. if not line.startswith('d'):
  171. # -rw-r--r-- 1 ftp ftp 476 Sep 1 2011 md5sum.txt
  172. # Last (9th) element is 'md5sum.txt' in the above example, but there
  173. # may be variations, so we discard only the first 8 entries.
  174. yield line.split(None, 8)[-1]
  175. def parse_html_filelist(s):
  176. re_href = re.compile(
  177. r'''<a[^>]*\shref=["']([^;?"']+?)[;?"']''',
  178. re.I|re.M)
  179. links = set(re_href.findall(s))
  180. for link in links:
  181. if not link.endswith('/'):
  182. yield unquote(link)
  183. def tryint(s):
  184. try:
  185. return int(s)
  186. except ValueError:
  187. return s
  188. @contextmanager
  189. def py2_tarxz(filename):
  190. import tempfile
  191. with tempfile.TemporaryFile() as tmp:
  192. subprocess.check_call(["xz", "-dc", filename], stdout=tmp.fileno())
  193. tmp.seek(0)
  194. with closing(tarfile.TarFile(fileobj=tmp)) as tf:
  195. yield tf
  196. def download_libxml2(dest_dir, version=None):
  197. """Downloads libxml2, returning the filename where the library was downloaded"""
  198. #version_re = re.compile(r'LATEST_LIBXML2_IS_([0-9.]+[0-9](?:-[abrc0-9]+)?)')
  199. version_re = re.compile(r'libxml2-([0-9.]+[0-9]).tar.xz')
  200. filename = 'libxml2-%s.tar.xz'
  201. if version == "2.9.12":
  202. # Temporarily using the latest master (2.9.12+) until there is a release that supports lxml again.
  203. from_location = "https://gitlab.gnome.org/GNOME/libxml2/-/archive/dea91c97debeac7c1aaf9c19f79029809e23a353/"
  204. version = "dea91c97debeac7c1aaf9c19f79029809e23a353"
  205. else:
  206. from_location = http_find_latest_version_directory(LIBXML2_LOCATION)
  207. return download_library(dest_dir, from_location, 'libxml2',
  208. version_re, filename, version=version)
  209. def download_libxslt(dest_dir, version=None):
  210. """Downloads libxslt, returning the filename where the library was downloaded"""
  211. #version_re = re.compile(r'LATEST_LIBXSLT_IS_([0-9.]+[0-9](?:-[abrc0-9]+)?)')
  212. version_re = re.compile(r'libxslt-([0-9.]+[0-9]).tar.xz')
  213. filename = 'libxslt-%s.tar.xz'
  214. from_location = http_find_latest_version_directory(LIBXSLT_LOCATION)
  215. return download_library(dest_dir, from_location, 'libxslt',
  216. version_re, filename, version=version)
  217. def download_libiconv(dest_dir, version=None):
  218. """Downloads libiconv, returning the filename where the library was downloaded"""
  219. version_re = re.compile(r'libiconv-([0-9.]+[0-9]).tar.gz')
  220. filename = 'libiconv-%s.tar.gz'
  221. return download_library(dest_dir, LIBICONV_LOCATION, 'libiconv',
  222. version_re, filename, version=version)
  223. def download_zlib(dest_dir, version):
  224. """Downloads zlib, returning the filename where the library was downloaded"""
  225. version_re = re.compile(r'zlib-([0-9.]+[0-9]).tar.gz')
  226. filename = 'zlib-%s.tar.gz'
  227. return download_library(dest_dir, ZLIB_LOCATION, 'zlib',
  228. version_re, filename, version=version)
  229. def find_max_version(libname, filenames, version_re=None):
  230. if version_re is None:
  231. version_re = re.compile(r'%s-([0-9.]+[0-9](?:-[abrc0-9]+)?)' % libname)
  232. versions = []
  233. for fn in filenames:
  234. match = version_re.search(fn)
  235. if match:
  236. version_string = match.group(1)
  237. versions.append((tuple(map(tryint, version_string.split('.'))),
  238. version_string))
  239. if not versions:
  240. raise Exception(
  241. "Could not find the most current version of %s from the files: %s" % (
  242. libname, filenames))
  243. versions.sort()
  244. version_string = versions[-1][-1]
  245. print('Latest version of %s is %s' % (libname, version_string))
  246. return version_string
  247. def download_library(dest_dir, location, name, version_re, filename, version=None):
  248. if version is None:
  249. try:
  250. if location.startswith('ftp://'):
  251. fns = remote_listdir(location)
  252. else:
  253. print(location)
  254. fns = http_listfiles(location, '(%s)' % filename.replace('%s', '(?:[0-9.]+[0-9])'))
  255. version = find_max_version(name, fns, version_re)
  256. except IOError:
  257. # network failure - maybe we have the files already?
  258. latest = (0,0,0)
  259. fns = os.listdir(dest_dir)
  260. for fn in fns:
  261. if fn.startswith(name+'-'):
  262. match = match_libfile_version(fn)
  263. if match:
  264. version_tuple = tuple(map(tryint, match.group(1).split('.')))
  265. if version_tuple > latest:
  266. latest = version_tuple
  267. filename = fn
  268. version = None
  269. if latest == (0,0,0):
  270. raise
  271. if version:
  272. filename = filename % version
  273. full_url = urljoin(location, filename)
  274. dest_filename = os.path.join(dest_dir, filename)
  275. if os.path.exists(dest_filename):
  276. print(('Using existing %s downloaded into %s '
  277. '(delete this file if you want to re-download the package)') % (
  278. name, dest_filename))
  279. else:
  280. print('Downloading %s into %s from %s' % (name, dest_filename, full_url))
  281. urlcleanup() # work around FTP bug 27973 in Py2.7.12
  282. urlretrieve(full_url, dest_filename)
  283. return dest_filename
  284. def unpack_tarball(tar_filename, dest):
  285. print('Unpacking %s into %s' % (os.path.basename(tar_filename), dest))
  286. if sys.version_info[0] < 3 and tar_filename.endswith('.xz'):
  287. # Py 2.7 lacks lzma support
  288. tar_cm = py2_tarxz(tar_filename)
  289. else:
  290. tar_cm = closing(tarfile.open(tar_filename))
  291. base_dir = None
  292. with tar_cm as tar:
  293. for member in tar:
  294. base_name = member.name.split('/')[0]
  295. if base_dir is None:
  296. base_dir = base_name
  297. elif base_dir != base_name:
  298. print('Unexpected path in %s: %s' % (tar_filename, base_name))
  299. tar.extractall(dest)
  300. return os.path.join(dest, base_dir)
  301. def call_subprocess(cmd, **kw):
  302. import subprocess
  303. cwd = kw.get('cwd', '.')
  304. cmd_desc = ' '.join(cmd)
  305. log.info('Running "%s" in %s' % (cmd_desc, cwd))
  306. returncode = subprocess.call(cmd, **kw)
  307. if returncode:
  308. raise Exception('Command "%s" returned code %s' % (cmd_desc, returncode))
  309. def safe_mkdir(dir):
  310. if not os.path.exists(dir):
  311. os.makedirs(dir)
  312. def cmmi(configure_cmd, build_dir, multicore=None, **call_setup):
  313. print('Starting build in %s' % build_dir)
  314. call_subprocess(configure_cmd, cwd=build_dir, **call_setup)
  315. if not multicore:
  316. make_jobs = multi_make_options
  317. elif int(multicore) > 1:
  318. make_jobs = ['-j%s' % multicore]
  319. else:
  320. make_jobs = []
  321. call_subprocess(
  322. ['make'] + make_jobs,
  323. cwd=build_dir, **call_setup)
  324. call_subprocess(
  325. ['make'] + make_jobs + ['install'],
  326. cwd=build_dir, **call_setup)
  327. def configure_darwin_env(env_setup):
  328. import platform
  329. # configure target architectures on MacOS-X (x86_64 only, by default)
  330. major_version, minor_version = tuple(map(int, platform.mac_ver()[0].split('.')[:2]))
  331. if major_version > 7:
  332. env_default = {
  333. 'CFLAGS': "-arch x86_64 -O2",
  334. 'LDFLAGS': "-arch x86_64",
  335. 'MACOSX_DEPLOYMENT_TARGET': "10.6"
  336. }
  337. env_default.update(os.environ)
  338. env_setup['env'] = env_default
  339. def build_libxml2xslt(download_dir, build_dir,
  340. static_include_dirs, static_library_dirs,
  341. static_cflags, static_binaries,
  342. libxml2_version=None,
  343. libxslt_version=None,
  344. libiconv_version=None,
  345. zlib_version=None,
  346. multicore=None):
  347. safe_mkdir(download_dir)
  348. safe_mkdir(build_dir)
  349. zlib_dir = unpack_tarball(download_zlib(download_dir, zlib_version), build_dir)
  350. libiconv_dir = unpack_tarball(download_libiconv(download_dir, libiconv_version), build_dir)
  351. libxml2_dir = unpack_tarball(download_libxml2(download_dir, libxml2_version), build_dir)
  352. libxslt_dir = unpack_tarball(download_libxslt(download_dir, libxslt_version), build_dir)
  353. prefix = os.path.join(os.path.abspath(build_dir), 'libxml2')
  354. lib_dir = os.path.join(prefix, 'lib')
  355. safe_mkdir(prefix)
  356. lib_names = ['libxml2', 'libexslt', 'libxslt', 'iconv', 'libz']
  357. existing_libs = {
  358. lib: os.path.join(lib_dir, filename)
  359. for lib in lib_names
  360. for filename in os.listdir(lib_dir)
  361. if lib in filename and filename.endswith('.a')
  362. } if os.path.isdir(lib_dir) else {}
  363. def has_current_lib(name, build_dir, _build_all_following=[False]):
  364. if _build_all_following[0]:
  365. return False # a dependency was rebuilt => rebuilt this lib as well
  366. lib_file = existing_libs.get(name)
  367. found = lib_file and os.path.getmtime(lib_file) > os.path.getmtime(build_dir)
  368. if found:
  369. print("Found pre-built '%s'" % name)
  370. else:
  371. # also rebuild all following libs (which may depend on this one)
  372. _build_all_following[0] = True
  373. return found
  374. call_setup = {}
  375. if sys.platform == 'darwin':
  376. configure_darwin_env(call_setup)
  377. configure_cmd = ['./configure',
  378. '--disable-dependency-tracking',
  379. '--disable-shared',
  380. '--prefix=%s' % prefix,
  381. ]
  382. # build zlib
  383. zlib_configure_cmd = [
  384. './configure',
  385. '--prefix=%s' % prefix,
  386. ]
  387. if not has_current_lib("libz", zlib_dir):
  388. cmmi(zlib_configure_cmd, zlib_dir, multicore, **call_setup)
  389. # build libiconv
  390. if not has_current_lib("iconv", libiconv_dir):
  391. cmmi(configure_cmd, libiconv_dir, multicore, **call_setup)
  392. # build libxml2
  393. libxml2_configure_cmd = configure_cmd + [
  394. '--without-python',
  395. '--with-iconv=%s' % prefix,
  396. '--with-zlib=%s' % prefix,
  397. ]
  398. if not libxml2_version:
  399. libxml2_version = os.path.basename(libxml2_dir).split('-', 1)[-1]
  400. if tuple(map(tryint, libxml2_version.split('-', 1)[0].split('.'))) >= (2, 9, 5):
  401. libxml2_configure_cmd.append('--without-lzma') # can't currently build that
  402. try:
  403. if tuple(map(tryint, libxml2_version.split('-', 1)[0].split('.'))) >= (2, 7, 3):
  404. libxml2_configure_cmd.append('--enable-rebuild-docs=no')
  405. except Exception:
  406. pass # this isn't required, so ignore any errors
  407. if not has_current_lib("libxml2", libxml2_dir):
  408. if not os.path.exists(os.path.join(libxml2_dir, "configure")):
  409. # Allow building from git sources by running autoconf etc.
  410. libxml2_configure_cmd[0] = "./autogen.sh"
  411. cmmi(libxml2_configure_cmd, libxml2_dir, multicore, **call_setup)
  412. # Fix up libxslt configure script (needed up to and including 1.1.34)
  413. # https://gitlab.gnome.org/GNOME/libxslt/-/commit/90c34c8bb90e095a8a8fe8b2ce368bd9ff1837cc
  414. with open(os.path.join(libxslt_dir, "configure"), 'rb') as f:
  415. config_script = f.read()
  416. if b' --libs print ' in config_script:
  417. config_script = config_script.replace(b' --libs print ', b' --libs ')
  418. with open(os.path.join(libxslt_dir, "configure"), 'wb') as f:
  419. f.write(config_script)
  420. # build libxslt
  421. libxslt_configure_cmd = configure_cmd + [
  422. '--without-python',
  423. '--with-libxml-prefix=%s' % prefix,
  424. '--without-crypto',
  425. ]
  426. if not (has_current_lib("libxslt", libxslt_dir) and has_current_lib("libexslt", libxslt_dir)):
  427. cmmi(libxslt_configure_cmd, libxslt_dir, multicore, **call_setup)
  428. # collect build setup for lxml
  429. xslt_config = os.path.join(prefix, 'bin', 'xslt-config')
  430. xml2_config = os.path.join(prefix, 'bin', 'xml2-config')
  431. static_include_dirs.extend([
  432. os.path.join(prefix, 'include'),
  433. os.path.join(prefix, 'include', 'libxml2'),
  434. os.path.join(prefix, 'include', 'libxslt'),
  435. os.path.join(prefix, 'include', 'libexslt')])
  436. static_library_dirs.append(lib_dir)
  437. listdir = os.listdir(lib_dir)
  438. static_binaries += [os.path.join(lib_dir, filename)
  439. for lib in lib_names
  440. for filename in listdir
  441. if lib in filename and filename.endswith('.a')]
  442. return xml2_config, xslt_config