buildlibxml.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389
  1. import os, re, sys, subprocess
  2. import tarfile
  3. from distutils import log, sysconfig, version
  4. try:
  5. from urlparse import urlsplit, urljoin
  6. from urllib import urlretrieve
  7. except ImportError:
  8. from urllib.parse import urlsplit, urljoin
  9. from urllib.request import urlretrieve
  10. multi_make_options = []
  11. try:
  12. import multiprocessing
  13. cpus = multiprocessing.cpu_count()
  14. if cpus > 1:
  15. if cpus > 5:
  16. cpus = 5
  17. multi_make_options = ['-j%d' % (cpus+1)]
  18. except:
  19. pass
  20. # use pre-built libraries on Windows
  21. def download_and_extract_zlatkovic_binaries(destdir):
  22. url = 'ftp://ftp.zlatkovic.com/pub/libxml/'
  23. libs = dict(
  24. libxml2 = None,
  25. libxslt = None,
  26. zlib = None,
  27. iconv = None,
  28. )
  29. for fn in ftp_listdir(url):
  30. for libname in libs:
  31. if fn.startswith(libname):
  32. assert libs[libname] is None, 'duplicate listings?'
  33. assert fn.endswith('.win32.zip')
  34. libs[libname] = fn
  35. if not os.path.exists(destdir): os.makedirs(destdir)
  36. for libname, libfn in libs.items():
  37. srcfile = urljoin(url, libfn)
  38. destfile = os.path.join(destdir, libfn)
  39. print('Retrieving "%s" to "%s"' % (srcfile, destfile))
  40. urlretrieve(srcfile, destfile)
  41. d = unpack_zipfile(destfile, destdir)
  42. libs[libname] = d
  43. return libs
  44. def find_top_dir_of_zipfile(zipfile):
  45. topdir = None
  46. files = [f.filename for f in zipfile.filelist]
  47. dirs = [d for d in files if d.endswith('/')]
  48. if dirs:
  49. dirs.sort(key=len)
  50. topdir = dirs[0]
  51. topdir = topdir[:topdir.index("/")+1]
  52. for path in files:
  53. if not path.startswith(topdir):
  54. topdir = None
  55. break
  56. assert topdir, (
  57. "cannot determine single top-level directory in zip file %s" %
  58. zipfile.filename)
  59. return topdir.rstrip('/')
  60. def unpack_zipfile(zipfn, destdir):
  61. assert zipfn.endswith('.zip')
  62. import zipfile
  63. print('Unpacking %s into %s' % (os.path.basename(zipfn), destdir))
  64. f = zipfile.ZipFile(zipfn)
  65. try:
  66. extracted_dir = os.path.join(destdir, find_top_dir_of_zipfile(f))
  67. f.extractall(path=destdir)
  68. finally:
  69. f.close()
  70. assert os.path.exists(extracted_dir), 'missing: %s' % extracted_dir
  71. return extracted_dir
  72. def get_prebuilt_libxml2xslt(download_dir, static_include_dirs, static_library_dirs):
  73. assert sys.platform.startswith('win')
  74. libs = download_and_extract_zlatkovic_binaries(download_dir)
  75. for libname, path in libs.items():
  76. i = os.path.join(path, 'include')
  77. l = os.path.join(path, 'lib')
  78. assert os.path.exists(i), 'does not exist: %s' % i
  79. assert os.path.exists(l), 'does not exist: %s' % l
  80. static_include_dirs.append(i)
  81. static_library_dirs.append(l)
  82. ## Routines to download and build libxml2/xslt from sources:
  83. LIBXML2_LOCATION = 'ftp://xmlsoft.org/libxml2/'
  84. LIBICONV_LOCATION = 'ftp://ftp.gnu.org/pub/gnu/libiconv/'
  85. match_libfile_version = re.compile('^[^-]*-([.0-9-]+)[.].*').match
  86. def ftp_listdir(url):
  87. import ftplib, posixpath
  88. scheme, netloc, path, qs, fragment = urlsplit(url)
  89. assert scheme.lower() == 'ftp'
  90. server = ftplib.FTP(netloc)
  91. server.login()
  92. files = [posixpath.basename(fn) for fn in server.nlst(path)]
  93. return files
  94. def tryint(s):
  95. try:
  96. return int(s)
  97. except ValueError:
  98. return s
  99. def download_libxml2(dest_dir, version=None):
  100. """Downloads libxml2, returning the filename where the library was downloaded"""
  101. version_re = re.compile(r'^LATEST_LIBXML2_IS_(.*)$')
  102. filename = 'libxml2-%s.tar.gz'
  103. return download_library(dest_dir, LIBXML2_LOCATION, 'libxml2',
  104. version_re, filename, version=version)
  105. def download_libxslt(dest_dir, version=None):
  106. """Downloads libxslt, returning the filename where the library was downloaded"""
  107. version_re = re.compile(r'^LATEST_LIBXSLT_IS_(.*)$')
  108. filename = 'libxslt-%s.tar.gz'
  109. return download_library(dest_dir, LIBXML2_LOCATION, 'libxslt',
  110. version_re, filename, version=version)
  111. def download_libiconv(dest_dir, version=None):
  112. """Downloads libiconv, returning the filename where the library was downloaded"""
  113. version_re = re.compile(r'^libiconv-([0-9.]+[0-9]).tar.gz$')
  114. filename = 'libiconv-%s.tar.gz'
  115. return download_library(dest_dir, LIBICONV_LOCATION, 'libiconv',
  116. version_re, filename, version=version)
  117. def download_library(dest_dir, location, name, version_re, filename,
  118. version=None):
  119. if version is None:
  120. try:
  121. fns = ftp_listdir(location)
  122. versions = []
  123. for fn in fns:
  124. match = version_re.search(fn)
  125. if match:
  126. version_string = match.group(1)
  127. versions.append((tuple(map(tryint, version_string.split('.'))),
  128. version_string))
  129. if versions:
  130. versions.sort()
  131. version = versions[-1][-1]
  132. print('Latest version of %s is %s' % (name, version))
  133. else:
  134. raise Exception(
  135. "Could not find the most current version of the %s from the files: %s"
  136. % (name, fns))
  137. except IOError:
  138. # network failure - maybe we have the files already?
  139. latest = (0,0,0)
  140. fns = os.listdir(dest_dir)
  141. for fn in fns:
  142. if fn.startswith(name+'-'):
  143. match = match_libfile_version(fn)
  144. if match:
  145. version_tuple = tuple(map(tryint, match.group(1).split('.')))
  146. if version_tuple > latest:
  147. latest = version_tuple
  148. filename = fn
  149. version = None
  150. if latest == (0,0,0):
  151. raise
  152. if version:
  153. filename = filename % version
  154. full_url = urljoin(location, filename)
  155. dest_filename = os.path.join(dest_dir, filename)
  156. if os.path.exists(dest_filename):
  157. print('Using existing %s downloaded into %s (delete this file if you want to re-download the package)'
  158. % (name, dest_filename))
  159. else:
  160. print('Downloading %s into %s' % (name, dest_filename))
  161. urlretrieve(full_url, dest_filename)
  162. return dest_filename
  163. ## Backported method of tarfile.TarFile.extractall (doesn't exist in 2.4):
  164. def _extractall(self, path=".", members=None):
  165. """Extract all members from the archive to the current working
  166. directory and set owner, modification time and permissions on
  167. directories afterwards. `path' specifies a different directory
  168. to extract to. `members' is optional and must be a subset of the
  169. list returned by getmembers().
  170. """
  171. import copy
  172. is_ignored_file = re.compile(
  173. r'''[\\/]((test|results?)[\\/]
  174. |doc[\\/].*(Log|[.](out|imp|err|ent|gif|tif|pdf))$
  175. |tests[\\/](.*[\\/])?(?!Makefile)[^\\/]*$
  176. |python[\\/].*[.]py$
  177. )
  178. ''', re.X).search
  179. directories = []
  180. if members is None:
  181. members = self
  182. for tarinfo in members:
  183. if is_ignored_file(tarinfo.name):
  184. continue
  185. if tarinfo.isdir():
  186. # Extract directories with a safe mode.
  187. directories.append((tarinfo.name, tarinfo))
  188. tarinfo = copy.copy(tarinfo)
  189. tarinfo.mode = 448 # 0700
  190. self.extract(tarinfo, path)
  191. # Reverse sort directories.
  192. directories.sort()
  193. directories.reverse()
  194. # Set correct owner, mtime and filemode on directories.
  195. for name, tarinfo in directories:
  196. dirpath = os.path.join(path, name)
  197. try:
  198. self.chown(tarinfo, dirpath)
  199. self.utime(tarinfo, dirpath)
  200. self.chmod(tarinfo, dirpath)
  201. except tarfile.ExtractError:
  202. if self.errorlevel > 1:
  203. raise
  204. else:
  205. self._dbg(1, "tarfile: %s" % sys.exc_info()[1])
  206. def unpack_tarball(tar_filename, dest):
  207. print('Unpacking %s into %s' % (os.path.basename(tar_filename), dest))
  208. tar = tarfile.open(tar_filename)
  209. base_dir = None
  210. for member in tar:
  211. base_name = member.name.split('/')[0]
  212. if base_dir is None:
  213. base_dir = base_name
  214. else:
  215. if base_dir != base_name:
  216. print('Unexpected path in %s: %s' % (tar_filename, base_name))
  217. _extractall(tar, dest)
  218. tar.close()
  219. return os.path.join(dest, base_dir)
  220. def call_subprocess(cmd, **kw):
  221. try:
  222. from subprocess import proc_call
  223. except ImportError:
  224. # no subprocess for Python 2.3
  225. def proc_call(cmd, **kwargs):
  226. cwd = kwargs.get('cwd', '.')
  227. old_cwd = os.getcwd()
  228. try:
  229. os.chdir(cwd)
  230. return os.system(' '.join(cmd))
  231. finally:
  232. os.chdir(old_cwd)
  233. cwd = kw.get('cwd', '.')
  234. cmd_desc = ' '.join(cmd)
  235. log.info('Running "%s" in %s' % (cmd_desc, cwd))
  236. returncode = proc_call(cmd, **kw)
  237. if returncode:
  238. raise Exception('Command "%s" returned code %s' % (cmd_desc, returncode))
  239. def safe_mkdir(dir):
  240. if not os.path.exists(dir):
  241. os.makedirs(dir)
  242. def cmmi(configure_cmd, build_dir, multicore=None, **call_setup):
  243. print('Starting build in %s' % build_dir)
  244. call_subprocess(configure_cmd, cwd=build_dir, **call_setup)
  245. if not multicore:
  246. make_jobs = multi_make_options
  247. elif int(multicore) > 1:
  248. make_jobs = ['-j%s' % multicore]
  249. else:
  250. make_jobs = []
  251. call_subprocess(
  252. ['make'] + make_jobs,
  253. cwd=build_dir, **call_setup)
  254. call_subprocess(
  255. ['make'] + make_jobs + ['install'],
  256. cwd=build_dir, **call_setup)
  257. def configure_darwin_env(env_setup):
  258. import platform
  259. # check target architectures on MacOS-X (ppc, i386, x86_64)
  260. major_version, minor_version = tuple(map(int, platform.mac_ver()[0].split('.')[:2]))
  261. if major_version > 7:
  262. # Check to see if ppc is supported (XCode4 drops ppc support)
  263. include_ppc = True
  264. if os.path.exists('/usr/bin/xcodebuild'):
  265. pipe = subprocess.Popen(['/usr/bin/xcodebuild', '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  266. out, _ = pipe.communicate()
  267. xcode_version = (out.decode('utf8').splitlines() or [''])[0]
  268. # Also parse only first digit, because 3.2.1 can't be parsed nicely
  269. if (xcode_version.startswith('Xcode') and
  270. version.StrictVersion(xcode_version.split()[1]) >= version.StrictVersion('4.0')):
  271. include_ppc = False
  272. arch_string = ""
  273. if include_ppc:
  274. arch_string = "-arch ppc "
  275. if minor_version < 6:
  276. env_default = {
  277. 'CFLAGS' : arch_string + "-arch i386 -isysroot /Developer/SDKs/MacOSX10.4u.sdk -O2",
  278. 'LDFLAGS' : arch_string + "-arch i386 -isysroot /Developer/SDKs/MacOSX10.4u.sdk",
  279. 'MACOSX_DEPLOYMENT_TARGET' : "10.3"
  280. }
  281. else:
  282. env_default = {
  283. 'CFLAGS' : arch_string + "-arch i386 -arch x86_64 -O2",
  284. 'LDFLAGS' : arch_string + "-arch i386 -arch x86_64",
  285. 'MACOSX_DEPLOYMENT_TARGET' : "10.6"
  286. }
  287. env = os.environ.copy()
  288. env_default.update(env)
  289. env_setup['env'] = env_default
  290. def build_libxml2xslt(download_dir, build_dir,
  291. static_include_dirs, static_library_dirs,
  292. static_cflags, static_binaries,
  293. libxml2_version=None, libxslt_version=None, libiconv_version=None,
  294. multicore=None):
  295. safe_mkdir(download_dir)
  296. safe_mkdir(build_dir)
  297. libiconv_dir = unpack_tarball(download_libiconv(download_dir, libiconv_version), build_dir)
  298. libxml2_dir = unpack_tarball(download_libxml2(download_dir, libxml2_version), build_dir)
  299. libxslt_dir = unpack_tarball(download_libxslt(download_dir, libxslt_version), build_dir)
  300. prefix = os.path.join(os.path.abspath(build_dir), 'libxml2')
  301. safe_mkdir(prefix)
  302. call_setup = {}
  303. if sys.platform == 'darwin':
  304. configure_darwin_env(call_setup)
  305. configure_cmd = ['./configure',
  306. '--disable-dependency-tracking',
  307. '--disable-shared',
  308. '--prefix=%s' % prefix,
  309. ]
  310. # build libiconv
  311. cmmi(configure_cmd, libiconv_dir, multicore, **call_setup)
  312. # build libxml2
  313. libxml2_configure_cmd = configure_cmd + [
  314. '--without-python',
  315. '--with-iconv=%s' % prefix]
  316. try:
  317. if libxml2_version and tuple(map(tryint, libxml2_version.split('.'))) >= (2,7,3):
  318. libxml2_configure_cmd.append('--enable-rebuild-docs=no')
  319. except Exception:
  320. pass # this isn't required, so ignore any errors
  321. cmmi(libxml2_configure_cmd, libxml2_dir, multicore, **call_setup)
  322. # build libxslt
  323. libxslt_configure_cmd = configure_cmd + [
  324. '--without-python',
  325. '--with-libxml-prefix=%s' % prefix,
  326. ]
  327. if sys.platform in ('darwin',):
  328. libxslt_configure_cmd += [
  329. '--without-crypto',
  330. ]
  331. cmmi(libxslt_configure_cmd, libxslt_dir, multicore, **call_setup)
  332. # collect build setup for lxml
  333. xslt_config = os.path.join(prefix, 'bin', 'xslt-config')
  334. xml2_config = os.path.join(prefix, 'bin', 'xml2-config')
  335. lib_dir = os.path.join(prefix, 'lib')
  336. static_include_dirs.extend([
  337. os.path.join(prefix, 'include'),
  338. os.path.join(prefix, 'include', 'libxml2'),
  339. os.path.join(prefix, 'include', 'libxslt'),
  340. os.path.join(prefix, 'include', 'libexslt')])
  341. static_library_dirs.append(lib_dir)
  342. for filename in os.listdir(lib_dir):
  343. if [l for l in ['iconv', 'libxml2', 'libxslt', 'libexslt'] if l in filename]:
  344. if [ext for ext in ['.a'] if filename.endswith(ext)]:
  345. static_binaries.append(os.path.join(lib_dir,filename))
  346. return (xml2_config, xslt_config)