verifier.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313
  1. import sys, os, binascii, shutil, io
  2. from . import __version_verifier_modules__
  3. from . import ffiplatform
  4. if sys.version_info >= (3, 3):
  5. import importlib.machinery
  6. def _extension_suffixes():
  7. return importlib.machinery.EXTENSION_SUFFIXES[:]
  8. else:
  9. import imp
  10. def _extension_suffixes():
  11. return [suffix for suffix, _, type in imp.get_suffixes()
  12. if type == imp.C_EXTENSION]
  13. if sys.version_info >= (3,):
  14. NativeIO = io.StringIO
  15. else:
  16. class NativeIO(io.BytesIO):
  17. def write(self, s):
  18. if isinstance(s, unicode):
  19. s = s.encode('ascii')
  20. super(NativeIO, self).write(s)
  21. def _hack_at_distutils():
  22. # Windows-only workaround for some configurations: see
  23. # https://bugs.python.org/issue23246 (Python 2.7 with
  24. # a specific MS compiler suite download)
  25. if sys.platform == "win32":
  26. try:
  27. import setuptools # for side-effects, patches distutils
  28. except ImportError:
  29. pass
  30. class Verifier(object):
  31. def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
  32. ext_package=None, tag='', force_generic_engine=False,
  33. source_extension='.c', flags=None, relative_to=None, **kwds):
  34. if ffi._parser._uses_new_feature:
  35. raise ffiplatform.VerificationError(
  36. "feature not supported with ffi.verify(), but only "
  37. "with ffi.set_source(): %s" % (ffi._parser._uses_new_feature,))
  38. self.ffi = ffi
  39. self.preamble = preamble
  40. if not modulename:
  41. flattened_kwds = ffiplatform.flatten(kwds)
  42. vengine_class = _locate_engine_class(ffi, force_generic_engine)
  43. self._vengine = vengine_class(self)
  44. self._vengine.patch_extension_kwds(kwds)
  45. self.flags = flags
  46. self.kwds = self.make_relative_to(kwds, relative_to)
  47. #
  48. if modulename:
  49. if tag:
  50. raise TypeError("can't specify both 'modulename' and 'tag'")
  51. else:
  52. key = '\x00'.join([sys.version[:3], __version_verifier_modules__,
  53. preamble, flattened_kwds] +
  54. ffi._cdefsources)
  55. if sys.version_info >= (3,):
  56. key = key.encode('utf-8')
  57. k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff)
  58. k1 = k1.lstrip('0x').rstrip('L')
  59. k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff)
  60. k2 = k2.lstrip('0').rstrip('L')
  61. modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key,
  62. k1, k2)
  63. suffix = _get_so_suffixes()[0]
  64. self.tmpdir = tmpdir or _caller_dir_pycache()
  65. self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension)
  66. self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
  67. self.ext_package = ext_package
  68. self._has_source = False
  69. self._has_module = False
  70. def write_source(self, file=None):
  71. """Write the C source code. It is produced in 'self.sourcefilename',
  72. which can be tweaked beforehand."""
  73. with self.ffi._lock:
  74. if self._has_source and file is None:
  75. raise ffiplatform.VerificationError(
  76. "source code already written")
  77. self._write_source(file)
  78. def compile_module(self):
  79. """Write the C source code (if not done already) and compile it.
  80. This produces a dynamic link library in 'self.modulefilename'."""
  81. with self.ffi._lock:
  82. if self._has_module:
  83. raise ffiplatform.VerificationError("module already compiled")
  84. if not self._has_source:
  85. self._write_source()
  86. self._compile_module()
  87. def load_library(self):
  88. """Get a C module from this Verifier instance.
  89. Returns an instance of a FFILibrary class that behaves like the
  90. objects returned by ffi.dlopen(), but that delegates all
  91. operations to the C module. If necessary, the C code is written
  92. and compiled first.
  93. """
  94. with self.ffi._lock:
  95. if not self._has_module:
  96. self._locate_module()
  97. if not self._has_module:
  98. if not self._has_source:
  99. self._write_source()
  100. self._compile_module()
  101. return self._load_library()
  102. def get_module_name(self):
  103. basename = os.path.basename(self.modulefilename)
  104. # kill both the .so extension and the other .'s, as introduced
  105. # by Python 3: 'basename.cpython-33m.so'
  106. basename = basename.split('.', 1)[0]
  107. # and the _d added in Python 2 debug builds --- but try to be
  108. # conservative and not kill a legitimate _d
  109. if basename.endswith('_d') and hasattr(sys, 'gettotalrefcount'):
  110. basename = basename[:-2]
  111. return basename
  112. def get_extension(self):
  113. _hack_at_distutils() # backward compatibility hack
  114. if not self._has_source:
  115. with self.ffi._lock:
  116. if not self._has_source:
  117. self._write_source()
  118. sourcename = ffiplatform.maybe_relative_path(self.sourcefilename)
  119. modname = self.get_module_name()
  120. return ffiplatform.get_extension(sourcename, modname, **self.kwds)
  121. def generates_python_module(self):
  122. return self._vengine._gen_python_module
  123. def make_relative_to(self, kwds, relative_to):
  124. if relative_to and os.path.dirname(relative_to):
  125. dirname = os.path.dirname(relative_to)
  126. kwds = kwds.copy()
  127. for key in ffiplatform.LIST_OF_FILE_NAMES:
  128. if key in kwds:
  129. lst = kwds[key]
  130. if not isinstance(lst, (list, tuple)):
  131. raise TypeError("keyword '%s' should be a list or tuple"
  132. % (key,))
  133. lst = [os.path.join(dirname, fn) for fn in lst]
  134. kwds[key] = lst
  135. return kwds
  136. # ----------
  137. def _locate_module(self):
  138. if not os.path.isfile(self.modulefilename):
  139. if self.ext_package:
  140. try:
  141. pkg = __import__(self.ext_package, None, None, ['__doc__'])
  142. except ImportError:
  143. return # cannot import the package itself, give up
  144. # (e.g. it might be called differently before installation)
  145. path = pkg.__path__
  146. else:
  147. path = None
  148. filename = self._vengine.find_module(self.get_module_name(), path,
  149. _get_so_suffixes())
  150. if filename is None:
  151. return
  152. self.modulefilename = filename
  153. self._vengine.collect_types()
  154. self._has_module = True
  155. def _write_source_to(self, file):
  156. self._vengine._f = file
  157. try:
  158. self._vengine.write_source_to_f()
  159. finally:
  160. del self._vengine._f
  161. def _write_source(self, file=None):
  162. if file is not None:
  163. self._write_source_to(file)
  164. else:
  165. # Write our source file to an in memory file.
  166. f = NativeIO()
  167. self._write_source_to(f)
  168. source_data = f.getvalue()
  169. # Determine if this matches the current file
  170. if os.path.exists(self.sourcefilename):
  171. with open(self.sourcefilename, "r") as fp:
  172. needs_written = not (fp.read() == source_data)
  173. else:
  174. needs_written = True
  175. # Actually write the file out if it doesn't match
  176. if needs_written:
  177. _ensure_dir(self.sourcefilename)
  178. with open(self.sourcefilename, "w") as fp:
  179. fp.write(source_data)
  180. # Set this flag
  181. self._has_source = True
  182. def _compile_module(self):
  183. # compile this C source
  184. tmpdir = os.path.dirname(self.sourcefilename)
  185. outputfilename = ffiplatform.compile(tmpdir, self.get_extension())
  186. try:
  187. same = ffiplatform.samefile(outputfilename, self.modulefilename)
  188. except OSError:
  189. same = False
  190. if not same:
  191. _ensure_dir(self.modulefilename)
  192. shutil.move(outputfilename, self.modulefilename)
  193. self._has_module = True
  194. def _load_library(self):
  195. assert self._has_module
  196. if self.flags is not None:
  197. return self._vengine.load_library(self.flags)
  198. else:
  199. return self._vengine.load_library()
  200. # ____________________________________________________________
  201. _FORCE_GENERIC_ENGINE = False # for tests
  202. def _locate_engine_class(ffi, force_generic_engine):
  203. if _FORCE_GENERIC_ENGINE:
  204. force_generic_engine = True
  205. if not force_generic_engine:
  206. if '__pypy__' in sys.builtin_module_names:
  207. force_generic_engine = True
  208. else:
  209. try:
  210. import _cffi_backend
  211. except ImportError:
  212. _cffi_backend = '?'
  213. if ffi._backend is not _cffi_backend:
  214. force_generic_engine = True
  215. if force_generic_engine:
  216. from . import vengine_gen
  217. return vengine_gen.VGenericEngine
  218. else:
  219. from . import vengine_cpy
  220. return vengine_cpy.VCPythonEngine
  221. # ____________________________________________________________
  222. _TMPDIR = None
  223. def _caller_dir_pycache():
  224. if _TMPDIR:
  225. return _TMPDIR
  226. result = os.environ.get('CFFI_TMPDIR')
  227. if result:
  228. return result
  229. filename = sys._getframe(2).f_code.co_filename
  230. return os.path.abspath(os.path.join(os.path.dirname(filename),
  231. '__pycache__'))
  232. def set_tmpdir(dirname):
  233. """Set the temporary directory to use instead of __pycache__."""
  234. global _TMPDIR
  235. _TMPDIR = dirname
  236. def cleanup_tmpdir(tmpdir=None, keep_so=False):
  237. """Clean up the temporary directory by removing all files in it
  238. called `_cffi_*.{c,so}` as well as the `build` subdirectory."""
  239. tmpdir = tmpdir or _caller_dir_pycache()
  240. try:
  241. filelist = os.listdir(tmpdir)
  242. except OSError:
  243. return
  244. if keep_so:
  245. suffix = '.c' # only remove .c files
  246. else:
  247. suffix = _get_so_suffixes()[0].lower()
  248. for fn in filelist:
  249. if fn.lower().startswith('_cffi_') and (
  250. fn.lower().endswith(suffix) or fn.lower().endswith('.c')):
  251. try:
  252. os.unlink(os.path.join(tmpdir, fn))
  253. except OSError:
  254. pass
  255. clean_dir = [os.path.join(tmpdir, 'build')]
  256. for dir in clean_dir:
  257. try:
  258. for fn in os.listdir(dir):
  259. fn = os.path.join(dir, fn)
  260. if os.path.isdir(fn):
  261. clean_dir.append(fn)
  262. else:
  263. os.unlink(fn)
  264. except OSError:
  265. pass
  266. def _get_so_suffixes():
  267. suffixes = _extension_suffixes()
  268. if not suffixes:
  269. # bah, no C_EXTENSION available. Occurs on pypy without cpyext
  270. if sys.platform == 'win32':
  271. suffixes = [".pyd"]
  272. else:
  273. suffixes = [".so"]
  274. return suffixes
  275. def _ensure_dir(filename):
  276. try:
  277. os.makedirs(os.path.dirname(filename))
  278. except OSError:
  279. pass