verifier.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316
  1. #
  2. # DEPRECATED: implementation for ffi.verify()
  3. #
  4. import sys, os, binascii, shutil, io
  5. from . import __version_verifier_modules__
  6. from . import ffiplatform
  7. if sys.version_info >= (3, 3):
  8. import importlib.machinery
  9. def _extension_suffixes():
  10. return importlib.machinery.EXTENSION_SUFFIXES[:]
  11. else:
  12. import imp
  13. def _extension_suffixes():
  14. return [suffix for suffix, _, type in imp.get_suffixes()
  15. if type == imp.C_EXTENSION]
  16. if sys.version_info >= (3,):
  17. NativeIO = io.StringIO
  18. else:
  19. class NativeIO(io.BytesIO):
  20. def write(self, s):
  21. if isinstance(s, unicode):
  22. s = s.encode('ascii')
  23. super(NativeIO, self).write(s)
  24. def _hack_at_distutils():
  25. # Windows-only workaround for some configurations: see
  26. # https://bugs.python.org/issue23246 (Python 2.7 with
  27. # a specific MS compiler suite download)
  28. if sys.platform == "win32":
  29. try:
  30. import setuptools # for side-effects, patches distutils
  31. except ImportError:
  32. pass
  33. class Verifier(object):
  34. def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
  35. ext_package=None, tag='', force_generic_engine=False,
  36. source_extension='.c', flags=None, relative_to=None, **kwds):
  37. if ffi._parser._uses_new_feature:
  38. raise ffiplatform.VerificationError(
  39. "feature not supported with ffi.verify(), but only "
  40. "with ffi.set_source(): %s" % (ffi._parser._uses_new_feature,))
  41. self.ffi = ffi
  42. self.preamble = preamble
  43. if not modulename:
  44. flattened_kwds = ffiplatform.flatten(kwds)
  45. vengine_class = _locate_engine_class(ffi, force_generic_engine)
  46. self._vengine = vengine_class(self)
  47. self._vengine.patch_extension_kwds(kwds)
  48. self.flags = flags
  49. self.kwds = self.make_relative_to(kwds, relative_to)
  50. #
  51. if modulename:
  52. if tag:
  53. raise TypeError("can't specify both 'modulename' and 'tag'")
  54. else:
  55. key = '\x00'.join([sys.version[:3], __version_verifier_modules__,
  56. preamble, flattened_kwds] +
  57. ffi._cdefsources)
  58. if sys.version_info >= (3,):
  59. key = key.encode('utf-8')
  60. k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff)
  61. k1 = k1.lstrip('0x').rstrip('L')
  62. k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff)
  63. k2 = k2.lstrip('0').rstrip('L')
  64. modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key,
  65. k1, k2)
  66. suffix = _get_so_suffixes()[0]
  67. self.tmpdir = tmpdir or _caller_dir_pycache()
  68. self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension)
  69. self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
  70. self.ext_package = ext_package
  71. self._has_source = False
  72. self._has_module = False
  73. def write_source(self, file=None):
  74. """Write the C source code. It is produced in 'self.sourcefilename',
  75. which can be tweaked beforehand."""
  76. with self.ffi._lock:
  77. if self._has_source and file is None:
  78. raise ffiplatform.VerificationError(
  79. "source code already written")
  80. self._write_source(file)
  81. def compile_module(self):
  82. """Write the C source code (if not done already) and compile it.
  83. This produces a dynamic link library in 'self.modulefilename'."""
  84. with self.ffi._lock:
  85. if self._has_module:
  86. raise ffiplatform.VerificationError("module already compiled")
  87. if not self._has_source:
  88. self._write_source()
  89. self._compile_module()
  90. def load_library(self):
  91. """Get a C module from this Verifier instance.
  92. Returns an instance of a FFILibrary class that behaves like the
  93. objects returned by ffi.dlopen(), but that delegates all
  94. operations to the C module. If necessary, the C code is written
  95. and compiled first.
  96. """
  97. with self.ffi._lock:
  98. if not self._has_module:
  99. self._locate_module()
  100. if not self._has_module:
  101. if not self._has_source:
  102. self._write_source()
  103. self._compile_module()
  104. return self._load_library()
  105. def get_module_name(self):
  106. basename = os.path.basename(self.modulefilename)
  107. # kill both the .so extension and the other .'s, as introduced
  108. # by Python 3: 'basename.cpython-33m.so'
  109. basename = basename.split('.', 1)[0]
  110. # and the _d added in Python 2 debug builds --- but try to be
  111. # conservative and not kill a legitimate _d
  112. if basename.endswith('_d') and hasattr(sys, 'gettotalrefcount'):
  113. basename = basename[:-2]
  114. return basename
  115. def get_extension(self):
  116. _hack_at_distutils() # backward compatibility hack
  117. if not self._has_source:
  118. with self.ffi._lock:
  119. if not self._has_source:
  120. self._write_source()
  121. sourcename = ffiplatform.maybe_relative_path(self.sourcefilename)
  122. modname = self.get_module_name()
  123. return ffiplatform.get_extension(sourcename, modname, **self.kwds)
  124. def generates_python_module(self):
  125. return self._vengine._gen_python_module
  126. def make_relative_to(self, kwds, relative_to):
  127. if relative_to and os.path.dirname(relative_to):
  128. dirname = os.path.dirname(relative_to)
  129. kwds = kwds.copy()
  130. for key in ffiplatform.LIST_OF_FILE_NAMES:
  131. if key in kwds:
  132. lst = kwds[key]
  133. if not isinstance(lst, (list, tuple)):
  134. raise TypeError("keyword '%s' should be a list or tuple"
  135. % (key,))
  136. lst = [os.path.join(dirname, fn) for fn in lst]
  137. kwds[key] = lst
  138. return kwds
  139. # ----------
  140. def _locate_module(self):
  141. if not os.path.isfile(self.modulefilename):
  142. if self.ext_package:
  143. try:
  144. pkg = __import__(self.ext_package, None, None, ['__doc__'])
  145. except ImportError:
  146. return # cannot import the package itself, give up
  147. # (e.g. it might be called differently before installation)
  148. path = pkg.__path__
  149. else:
  150. path = None
  151. filename = self._vengine.find_module(self.get_module_name(), path,
  152. _get_so_suffixes())
  153. if filename is None:
  154. return
  155. self.modulefilename = filename
  156. self._vengine.collect_types()
  157. self._has_module = True
  158. def _write_source_to(self, file):
  159. self._vengine._f = file
  160. try:
  161. self._vengine.write_source_to_f()
  162. finally:
  163. del self._vengine._f
  164. def _write_source(self, file=None):
  165. if file is not None:
  166. self._write_source_to(file)
  167. else:
  168. # Write our source file to an in memory file.
  169. f = NativeIO()
  170. self._write_source_to(f)
  171. source_data = f.getvalue()
  172. # Determine if this matches the current file
  173. if os.path.exists(self.sourcefilename):
  174. with open(self.sourcefilename, "r") as fp:
  175. needs_written = not (fp.read() == source_data)
  176. else:
  177. needs_written = True
  178. # Actually write the file out if it doesn't match
  179. if needs_written:
  180. _ensure_dir(self.sourcefilename)
  181. with open(self.sourcefilename, "w") as fp:
  182. fp.write(source_data)
  183. # Set this flag
  184. self._has_source = True
  185. def _compile_module(self):
  186. # compile this C source
  187. tmpdir = os.path.dirname(self.sourcefilename)
  188. outputfilename = ffiplatform.compile(tmpdir, self.get_extension())
  189. try:
  190. same = ffiplatform.samefile(outputfilename, self.modulefilename)
  191. except OSError:
  192. same = False
  193. if not same:
  194. _ensure_dir(self.modulefilename)
  195. shutil.move(outputfilename, self.modulefilename)
  196. self._has_module = True
  197. def _load_library(self):
  198. assert self._has_module
  199. if self.flags is not None:
  200. return self._vengine.load_library(self.flags)
  201. else:
  202. return self._vengine.load_library()
  203. # ____________________________________________________________
  204. _FORCE_GENERIC_ENGINE = False # for tests
  205. def _locate_engine_class(ffi, force_generic_engine):
  206. if _FORCE_GENERIC_ENGINE:
  207. force_generic_engine = True
  208. if not force_generic_engine:
  209. if '__pypy__' in sys.builtin_module_names:
  210. force_generic_engine = True
  211. else:
  212. try:
  213. import _cffi_backend
  214. except ImportError:
  215. _cffi_backend = '?'
  216. if ffi._backend is not _cffi_backend:
  217. force_generic_engine = True
  218. if force_generic_engine:
  219. from . import vengine_gen
  220. return vengine_gen.VGenericEngine
  221. else:
  222. from . import vengine_cpy
  223. return vengine_cpy.VCPythonEngine
  224. # ____________________________________________________________
  225. _TMPDIR = None
  226. def _caller_dir_pycache():
  227. if _TMPDIR:
  228. return _TMPDIR
  229. result = os.environ.get('CFFI_TMPDIR')
  230. if result:
  231. return result
  232. filename = sys._getframe(2).f_code.co_filename
  233. return os.path.abspath(os.path.join(os.path.dirname(filename),
  234. '__pycache__'))
  235. def set_tmpdir(dirname):
  236. """Set the temporary directory to use instead of __pycache__."""
  237. global _TMPDIR
  238. _TMPDIR = dirname
  239. def cleanup_tmpdir(tmpdir=None, keep_so=False):
  240. """Clean up the temporary directory by removing all files in it
  241. called `_cffi_*.{c,so}` as well as the `build` subdirectory."""
  242. tmpdir = tmpdir or _caller_dir_pycache()
  243. try:
  244. filelist = os.listdir(tmpdir)
  245. except OSError:
  246. return
  247. if keep_so:
  248. suffix = '.c' # only remove .c files
  249. else:
  250. suffix = _get_so_suffixes()[0].lower()
  251. for fn in filelist:
  252. if fn.lower().startswith('_cffi_') and (
  253. fn.lower().endswith(suffix) or fn.lower().endswith('.c')):
  254. try:
  255. os.unlink(os.path.join(tmpdir, fn))
  256. except OSError:
  257. pass
  258. clean_dir = [os.path.join(tmpdir, 'build')]
  259. for dir in clean_dir:
  260. try:
  261. for fn in os.listdir(dir):
  262. fn = os.path.join(dir, fn)
  263. if os.path.isdir(fn):
  264. clean_dir.append(fn)
  265. else:
  266. os.unlink(fn)
  267. except OSError:
  268. pass
  269. def _get_so_suffixes():
  270. suffixes = _extension_suffixes()
  271. if not suffixes:
  272. # bah, no C_EXTENSION available. Occurs on pypy without cpyext
  273. if sys.platform == 'win32':
  274. suffixes = [".pyd"]
  275. else:
  276. suffixes = [".so"]
  277. return suffixes
  278. def _ensure_dir(filename):
  279. try:
  280. os.makedirs(os.path.dirname(filename))
  281. except OSError:
  282. pass