idna-data 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671
  1. #!/usr/bin/env python3
  2. import argparse, collections, datetime, os, re, sys, unicodedata
  3. from urllib.request import urlopen
  4. from intranges import intranges_from_list
  5. if sys.version_info[0] < 3:
  6. print("Only Python 3 supported.")
  7. sys.exit(2)
  8. # PREFERRED_VERSION = 'latest' # https://github.com/kjd/idna/issues/8
  9. PREFERRED_VERSION = '6.3.0'
  10. UCD_URL = 'http://www.unicode.org/Public/{version}/ucd/{filename}'
  11. UTS46_URL = 'http://www.unicode.org/Public/idna/{version}/{filename}'
  12. DEFAULT_CACHE_DIR = '~/.cache/unidata'
  13. # Scripts affected by IDNA contextual rules
  14. SCRIPT_WHITELIST = sorted(['Greek', 'Han', 'Hebrew', 'Hiragana', 'Katakana'])
  15. # Used to piece apart UTS#46 data for Jython compatibility
  16. UTS46_SEGMENT_SIZE = 100
  17. UTS46_STATUSES = {
  18. "valid": ("V", False),
  19. "ignored": ("I", False),
  20. "mapped": ("M", True),
  21. "deviation": ("D", True),
  22. "disallowed": ("X", False),
  23. "disallowed_STD3_valid": ("3", False),
  24. "disallowed_STD3_mapped": ("3", True)
  25. }
  26. # Exceptions are manually assigned in Section 2.6 of RFC 5892.
  27. exceptions = {
  28. 0x00DF: 'PVALID', # LATIN SMALL LETTER SHARP S
  29. 0x03C2: 'PVALID', # GREEK SMALL LETTER FINAL SIGMA
  30. 0x06FD: 'PVALID', # ARABIC SIGN SINDHI AMPERSAND
  31. 0x06FE: 'PVALID', # ARABIC SIGN SINDHI POSTPOSITION MEN
  32. 0x0F0B: 'PVALID', # TIBETAN MARK INTERSYLLABIC TSHEG
  33. 0x3007: 'PVALID', # IDEOGRAPHIC NUMBER ZERO
  34. 0x00B7: 'CONTEXTO', # MIDDLE DOT
  35. 0x0375: 'CONTEXTO', # GREEK LOWER NUMERAL SIGN (KERAIA)
  36. 0x05F3: 'CONTEXTO', # HEBREW PUNCTUATION GERESH
  37. 0x05F4: 'CONTEXTO', # HEBREW PUNCTUATION GERSHAYIM
  38. 0x30FB: 'CONTEXTO', # KATAKANA MIDDLE DOT
  39. 0x0660: 'CONTEXTO', # ARABIC-INDIC DIGIT ZERO
  40. 0x0661: 'CONTEXTO', # ARABIC-INDIC DIGIT ONE
  41. 0x0662: 'CONTEXTO', # ARABIC-INDIC DIGIT TWO
  42. 0x0663: 'CONTEXTO', # ARABIC-INDIC DIGIT THREE
  43. 0x0664: 'CONTEXTO', # ARABIC-INDIC DIGIT FOUR
  44. 0x0665: 'CONTEXTO', # ARABIC-INDIC DIGIT FIVE
  45. 0x0666: 'CONTEXTO', # ARABIC-INDIC DIGIT SIX
  46. 0x0667: 'CONTEXTO', # ARABIC-INDIC DIGIT SEVEN
  47. 0x0668: 'CONTEXTO', # ARABIC-INDIC DIGIT EIGHT
  48. 0x0669: 'CONTEXTO', # ARABIC-INDIC DIGIT NINE
  49. 0x06F0: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT ZERO
  50. 0x06F1: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT ONE
  51. 0x06F2: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT TWO
  52. 0x06F3: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT THREE
  53. 0x06F4: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT FOUR
  54. 0x06F5: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT FIVE
  55. 0x06F6: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT SIX
  56. 0x06F7: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT SEVEN
  57. 0x06F8: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT EIGHT
  58. 0x06F9: 'CONTEXTO', # EXTENDED ARABIC-INDIC DIGIT NINE
  59. 0x0640: 'DISALLOWED', # ARABIC TATWEEL
  60. 0x07FA: 'DISALLOWED', # NKO LAJANYALAN
  61. 0x302E: 'DISALLOWED', # HANGUL SINGLE DOT TONE MARK
  62. 0x302F: 'DISALLOWED', # HANGUL DOUBLE DOT TONE MARK
  63. 0x3031: 'DISALLOWED', # VERTICAL KANA REPEAT MARK
  64. 0x3032: 'DISALLOWED', # VERTICAL KANA REPEAT WITH VOICED SOUND MARK
  65. 0x3033: 'DISALLOWED', # VERTICAL KANA REPEAT MARK UPPER HALF
  66. 0x3034: 'DISALLOWED', # VERTICAL KANA REPEAT WITH VOICED SOUND MARK UPPER HA
  67. 0x3035: 'DISALLOWED', # VERTICAL KANA REPEAT MARK LOWER HALF
  68. 0x303B: 'DISALLOWED', # VERTICAL IDEOGRAPHIC ITERATION MARK
  69. }
  70. backwardscompatible = {}
  71. def hexrange(start, end):
  72. return range(int(start, 16), int(end, 16) + 1)
  73. def hexvalue(value):
  74. return int(value, 16)
  75. class UnicodeVersion(object):
  76. def __init__(self, version):
  77. result = re.match('^(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)$', version)
  78. if result:
  79. self.major = int(result.group('major'))
  80. self.minor = int(result.group('minor'))
  81. self.patch = int(result.group('patch'))
  82. self.numerical = (self.major << 8) + (self.minor << 4) + self.patch
  83. self.latest = False
  84. elif version == 'latest':
  85. self.latest = True
  86. else:
  87. raise ValueError('Unrecognized Unicode version')
  88. def __repr__(self, with_date=True):
  89. if self.latest:
  90. if with_date:
  91. return 'latest@{}'.format(datetime.datetime.now().strftime('%Y-%m-%d'))
  92. else:
  93. return 'latest'
  94. else:
  95. return "{}.{}.{}".format(self.major, self.minor, self.patch)
  96. @property
  97. def tag(self):
  98. return self.__repr__(with_date=False)
  99. def __gt__(self, other):
  100. if self.latest:
  101. return True
  102. return self.numerical > other.numerical
  103. def __eq__(self, other):
  104. if self.latest:
  105. return False
  106. return self.numerical == other.numerical
  107. class UnicodeData(object):
  108. def __init__(self, version, cache, args):
  109. self.version = UnicodeVersion(version)
  110. self.system_version = UnicodeVersion(unicodedata.unidata_version)
  111. self.source = args.source
  112. self.cache = cache
  113. self.max = 0
  114. if self.system_version < self.version:
  115. print("Warning: Character stability not guaranteed as Python Unicode data {}"
  116. " older than requested {}".format(self.system_version, self.version))
  117. self._load_unicodedata()
  118. self._load_proplist()
  119. self._load_derivedcoreprops()
  120. self._load_blocks()
  121. self._load_casefolding()
  122. self._load_hangulst()
  123. self._load_arabicshaping()
  124. self._load_scripts()
  125. self._load_uts46mapping()
  126. def _load_unicodedata(self):
  127. f_ud = self._ucdfile('UnicodeData.txt')
  128. self.ucd_data = {}
  129. range_begin = None
  130. for line in f_ud.splitlines():
  131. fields = line.split(';')
  132. value = int(fields[0], 16)
  133. start_marker = re.match('^<(?P<name>.*?), First>$', fields[1])
  134. end_marker = re.match('^<(?P<name>.*?), Last>$', fields[1])
  135. if start_marker:
  136. range_begin = value
  137. elif end_marker:
  138. for i in range(range_begin, value+1):
  139. fields[1] = '<{}>'.format(end_marker.group('name'))
  140. self.ucd_data[i] = fields[1:]
  141. range_begin = None
  142. else:
  143. self.ucd_data[value] = fields[1:]
  144. def _load_proplist(self):
  145. f_pl = self._ucdfile('PropList.txt')
  146. self.ucd_props = collections.defaultdict(list)
  147. for line in f_pl.splitlines():
  148. result = re.match(
  149. '^(?P<start>[0-9A-F]{4,6})(|\.\.(?P<end>[0-9A-F]{4,6}))\s*;\s*(?P<prop>\S+)\s*(|\#.*)$',
  150. line)
  151. if result:
  152. if result.group('end'):
  153. for i in hexrange(result.group('start'), result.group('end')):
  154. self.ucd_props[i].append(result.group('prop'))
  155. else:
  156. i = hexvalue(result.group('start'))
  157. self.ucd_props[i].append(result.group('prop'))
  158. def _load_derivedcoreprops(self):
  159. f_dcp = self._ucdfile('DerivedCoreProperties.txt')
  160. for line in f_dcp.splitlines():
  161. result = re.match(
  162. '^(?P<start>[0-9A-F]{4,6})(|\.\.(?P<end>[0-9A-F]{4,6}))\s*;\s*(?P<prop>\S+)\s*(|\#.*)$',
  163. line)
  164. if result:
  165. if result.group('end'):
  166. for i in hexrange(result.group('start'), result.group('end')):
  167. self.ucd_props[i].append(result.group('prop'))
  168. else:
  169. i = hexvalue(result.group('start'))
  170. self.ucd_props[i].append(result.group('prop'))
  171. def _load_blocks(self):
  172. self.ucd_block = {}
  173. f_b = self._ucdfile('Blocks.txt')
  174. for line in f_b.splitlines():
  175. result = re.match(
  176. '^(?P<start>[0-9A-F]{4,6})\.\.(?P<end>[0-9A-F]{4,6})\s*;\s*(?P<block>.*)\s*$',
  177. line)
  178. if result:
  179. for i in hexrange(result.group('start'), result.group('end')):
  180. self.ucd_block[i] = result.group('block')
  181. self.max = max(self.max, i)
  182. def _load_casefolding(self):
  183. self.ucd_cf = {}
  184. f_cf = self._ucdfile('CaseFolding.txt')
  185. for line in f_cf.splitlines():
  186. result = re.match(
  187. '^(?P<cp>[0-9A-F]{4,6})\s*;\s*(?P<type>\S+)\s*;\s*(?P<subst>[0-9A-F\s]+)\s*',
  188. line)
  189. if result:
  190. if result.group('type') in ('C', 'F'):
  191. self.ucd_cf[int(result.group('cp'), 16)] = \
  192. ''.join([chr(int(x, 16)) for x in result.group('subst').split(' ')])
  193. def _load_hangulst(self):
  194. self.ucd_hst = {}
  195. f_hst = self._ucdfile('HangulSyllableType.txt')
  196. for line in f_hst.splitlines():
  197. result = re.match(
  198. '^(?P<start>[0-9A-F]{4,6})\.\.(?P<end>[0-9A-F]{4,6})\s*;\s*(?P<type>\S+)\s*(|\#.*)$',
  199. line)
  200. if result:
  201. for i in hexrange(result.group('start'), result.group('end')):
  202. self.ucd_hst[i] = result.group('type')
  203. def _load_arabicshaping(self):
  204. self.ucd_as = {}
  205. f_as = self._ucdfile('ArabicShaping.txt')
  206. for line in f_as.splitlines():
  207. result = re.match('^(?P<cp>[0-9A-F]{4,6})\s*;\s*.*?\s*;\s*(?P<jt>\S+)\s*;', line)
  208. if result:
  209. self.ucd_as[int(result.group('cp'), 16)] = result.group('jt')
  210. def _load_scripts(self):
  211. self.ucd_s = {}
  212. f_s = self._ucdfile('Scripts.txt')
  213. for line in f_s.splitlines():
  214. result = re.match(
  215. '^(?P<start>[0-9A-F]{4,6})(|\.\.(?P<end>[0-9A-F]{4,6}))\s*;\s*(?P<script>\S+)\s*(|\#.*)$',
  216. line)
  217. if result:
  218. if not result.group('script') in self.ucd_s:
  219. self.ucd_s[result.group('script')] = set()
  220. if result.group('end'):
  221. for i in hexrange(result.group('start'), result.group('end')):
  222. self.ucd_s[result.group('script')].add(i)
  223. else:
  224. i = hexvalue(result.group('start'))
  225. self.ucd_s[result.group('script')].add(i)
  226. def _load_uts46mapping(self):
  227. self.ucd_idnamt = {}
  228. f_idnamt = self._ucdfile('IdnaMappingTable.txt', urlbase=UTS46_URL)
  229. for line in f_idnamt.splitlines():
  230. result = re.match(
  231. '^(?P<start>[0-9A-F]{4,6})(|\.\.(?P<end>[0-9A-F]{4,6}))\s*;\s*(?P<fields>[^#]+)',
  232. line)
  233. if result:
  234. fields = [x.strip() for x in result.group('fields').split(';')]
  235. if result.group('end'):
  236. for i in hexrange(result.group('start'), result.group('end')):
  237. self.ucd_idnamt[i] = fields
  238. else:
  239. i = hexvalue(result.group('start'))
  240. self.ucd_idnamt[i] = fields
  241. def _ucdfile(self, filename, urlbase=UCD_URL):
  242. if self.source:
  243. f = open("{}/{}".format(self.source, filename))
  244. return f.read()
  245. else:
  246. cache_file = None
  247. if self.cache:
  248. cache_file = os.path.expanduser("{}/{}/{}".format(
  249. self.cache, self.version.tag, filename))
  250. if os.path.isfile(cache_file):
  251. f = open(cache_file)
  252. return f.read()
  253. version_path = self.version.tag
  254. if version_path == 'latest':
  255. version_path = 'UCD/latest'
  256. url = urlbase.format(
  257. version=version_path,
  258. filename=filename,
  259. )
  260. content = urlopen(url).read()
  261. if cache_file:
  262. if not os.path.isdir(os.path.dirname(cache_file)):
  263. os.makedirs(os.path.dirname(cache_file))
  264. f = open(cache_file, 'wb')
  265. f.write(content)
  266. f.close()
  267. return str(content)
  268. def codepoints(self):
  269. for i in range(0, self.max + 1):
  270. yield CodePoint(i, ucdata=self)
  271. class CodePoint:
  272. def __init__(self, value=None, ucdata=None):
  273. self.value = value
  274. self.ucdata = ucdata
  275. def _casefold(self, s):
  276. r = ''
  277. for c in s:
  278. r += self.ucdata.ucd_cf.get(ord(c), c)
  279. return r
  280. @property
  281. def exception_value(self):
  282. return exceptions.get(self.value, False)
  283. @property
  284. def compat_value(self):
  285. return backwardscompatible.get(self.value, False)
  286. @property
  287. def name(self):
  288. if self.value in self.ucdata.ucd_data:
  289. return self.ucdata.ucd_data[self.value][0]
  290. elif 'Noncharacter_Code_Point' in self.ucdata.ucd_props[self.value]:
  291. return '<noncharacter>'
  292. else:
  293. return '<reserved>'
  294. @property
  295. def general_category(self):
  296. return self.ucdata.ucd_data.get(self.value, [None, None])[1]
  297. @property
  298. def unassigned(self):
  299. return not ('Noncharacter_Code_Point' in self.ucdata.ucd_props[self.value] or \
  300. self.value in self.ucdata.ucd_data)
  301. @property
  302. def ldh(self):
  303. if self.value == 0x002d or \
  304. self.value in range(0x0030, 0x0039+1) or \
  305. self.value in range(0x0061, 0x007a+1):
  306. return True
  307. return False
  308. @property
  309. def join_control(self):
  310. return 'Join_Control' in self.ucdata.ucd_props[self.value]
  311. @property
  312. def joining_type(self):
  313. return self.ucdata.ucd_as.get(self.value, None)
  314. @property
  315. def char(self):
  316. return chr(self.value)
  317. @property
  318. def nfkc_cf(self):
  319. return unicodedata.normalize('NFKC',
  320. self._casefold(unicodedata.normalize('NFKC', self.char)))
  321. @property
  322. def unstable(self):
  323. return self.char != self.nfkc_cf
  324. @property
  325. def in_ignorableproperties(self):
  326. for prop in ['Default_Ignorable_Code_Point', 'White_Space', 'Noncharacter_Code_Point']:
  327. if prop in self.ucdata.ucd_props[self.value]:
  328. return True
  329. return False
  330. @property
  331. def in_ignorableblocks(self):
  332. return self.ucdata.ucd_block.get(self.value) in (
  333. 'Combining Diacritical Marks for Symbols', 'Musical Symbols',
  334. 'Ancient Greek Musical Notation'
  335. )
  336. @property
  337. def oldhanguljamo(self):
  338. return self.ucdata.ucd_hst.get(self.value) in ('L', 'V', 'T')
  339. @property
  340. def in_lettersdigits(self):
  341. return self.general_category in ('Ll', 'Lu', 'Lo', 'Nd', 'Lm', 'Mn', 'Mc')
  342. @property
  343. def idna2008_status(self):
  344. if self.exception_value:
  345. return self.exception_value
  346. elif self.compat_value:
  347. return self.compat_value
  348. elif self.unassigned:
  349. return 'UNASSIGNED'
  350. elif self.ldh:
  351. return 'PVALID'
  352. elif self.join_control:
  353. return 'CONTEXTJ'
  354. elif self.unstable:
  355. return 'DISALLOWED'
  356. elif self.in_ignorableproperties:
  357. return 'DISALLOWED'
  358. elif self.in_ignorableblocks:
  359. return 'DISALLOWED'
  360. elif self.oldhanguljamo:
  361. return 'DISALLOWED'
  362. elif self.in_lettersdigits:
  363. return 'PVALID'
  364. else:
  365. return 'DISALLOWED'
  366. @property
  367. def uts46_data(self):
  368. return self.ucdata.ucd_idnamt.get(self.value, None)
  369. @property
  370. def uts46_status(self):
  371. return ' '.join(self.uts46_data)
  372. def diagnose_codepoint(codepoint, args, ucdata):
  373. cp = CodePoint(codepoint, ucdata=ucdata)
  374. print("U+{:04X}:".format(codepoint))
  375. print(" Name: {}".format(cp.name))
  376. print("1 Exceptions: {}".format(exceptions.get(codepoint, False)))
  377. print("2 Backwards Compat: {}".format(backwardscompatible.get(codepoint, False)))
  378. print("3 Unassigned: {}".format(cp.unassigned))
  379. print("4 LDH: {}".format(cp.ldh))
  380. print(" Properties: {}".format(" ".join(sorted(ucdata.ucd_props.get(codepoint, ['None'])))))
  381. print("5 .Join Control: {}".format(cp.join_control))
  382. print(" NFKC CF: {}".format(" ".join(["U+{:04X}".format(ord(x)) for x in cp.nfkc_cf])))
  383. print("6 .Unstable: {}".format(cp.unstable))
  384. print("7 .Ignorable Prop: {}".format(cp.in_ignorableproperties))
  385. print(" Block: {}".format(ucdata.ucd_block.get(codepoint, None)))
  386. print("8 .Ignorable Block: {}".format(cp.in_ignorableblocks))
  387. print(" Hangul Syll Type: {}".format(ucdata.ucd_hst.get(codepoint, None)))
  388. print("9 .Old Hangul Jamo: {}".format(cp.oldhanguljamo))
  389. print(" General Category: {}".format(cp.general_category))
  390. print("10 .Letters Digits: {}".format(cp.in_lettersdigits))
  391. print("== IDNA 2008: {}".format(cp.idna2008_status))
  392. print("== UTS 46: {}".format(cp.uts46_status))
  393. print("(Unicode {} [sys:{}])".format(ucdata.version, ucdata.system_version))
  394. def ucdrange(start, end):
  395. if start == end:
  396. return ("{:04X}".format(start.value), start.name)
  397. else:
  398. return ("{:04X}..{:04X}".format(start.value, end.value),
  399. "{}..{}".format(start.name, end.name))
  400. def optimised_list(d):
  401. yield '('
  402. for value in intranges_from_list(d):
  403. yield ' {},'.format(hex(value))
  404. yield ' ),'
  405. def make_table(args, ucdata):
  406. last_status = None
  407. cps = []
  408. table_data = []
  409. for cp in ucdata.codepoints():
  410. status = cp.idna2008_status
  411. if (last_status and last_status != status):
  412. (values, description) = ucdrange(cps[0], cps[-1])
  413. table_data.append([values, last_status, description])
  414. cps = []
  415. last_status = status
  416. cps.append(cp)
  417. (values, description) = ucdrange(cps[0], cps[-1])
  418. table_data.append([values, last_status, description])
  419. if args.dir:
  420. f = open("{}/idna-table-{}.txt".format(args.dir, ucdata.version), 'wb')
  421. for row in table_data:
  422. f.write("{:12}; {:12}# {:.44}\n".format(*row).encode('ascii'))
  423. f.close()
  424. else:
  425. for row in table_data:
  426. print("{:12}; {:12}# {:.44}".format(*row))
  427. def idna_libdata(ucdata):
  428. yield "# This file is automatically generated by tools/idna-data\n"
  429. yield "__version__ = \"{}\"".format(ucdata.version)
  430. #
  431. # Script classifications are used by some CONTEXTO rules in RFC 5891
  432. #
  433. yield "scripts = {"
  434. for script in SCRIPT_WHITELIST:
  435. prefix = " '{0}': ".format(script)
  436. for line in optimised_list(ucdata.ucd_s[script]):
  437. yield prefix + line
  438. prefix = ""
  439. yield "}"
  440. #
  441. # Joining types are used by CONTEXTJ rule A.1
  442. #
  443. yield "joining_types = {"
  444. for cp in ucdata.codepoints():
  445. if cp.joining_type:
  446. yield " 0x{0:x}: {1},".format(cp.value, ord(cp.joining_type))
  447. yield "}"
  448. #
  449. # These are the classification of codepoints into PVALID, CONTEXTO, CONTEXTJ, etc.
  450. #
  451. yield "codepoint_classes = {"
  452. classes = {}
  453. for cp in ucdata.codepoints():
  454. status = cp.idna2008_status
  455. if status in ('UNASSIGNED', 'DISALLOWED'):
  456. continue
  457. if not status in classes:
  458. classes[status] = set()
  459. classes[status].add(cp.value)
  460. for status in ['PVALID', 'CONTEXTJ', 'CONTEXTO']:
  461. prefix = " '{0}': ".format(status)
  462. for line in optimised_list(classes[status]):
  463. yield prefix + line
  464. prefix = ""
  465. yield "}"
  466. def uts46_ranges(ucdata):
  467. last = (None, None)
  468. for cp in ucdata.codepoints():
  469. fields = cp.uts46_data
  470. if not fields:
  471. continue
  472. status, mapping = UTS46_STATUSES[fields[0]]
  473. if mapping:
  474. mapping = "".join(chr(int(codepoint, 16)) for codepoint in fields[1].split())
  475. mapping = mapping.replace("\\", "\\\\").replace("'", "\\'")
  476. else:
  477. mapping = None
  478. if cp.value > 255 and (status, mapping) == last:
  479. continue
  480. last = (status, mapping)
  481. if mapping is not None:
  482. yield "(0x{0:X}, '{1}', u'{2}')".format(cp.value, status, mapping)
  483. else:
  484. yield "(0x{0:X}, '{1}')".format(cp.value, status)
  485. def uts46_libdata(ucdata):
  486. yield "# This file is automatically generated by tools/idna-data"
  487. yield "# vim: set fileencoding=utf-8 :\n"
  488. yield '"""IDNA Mapping Table from UTS46."""\n\n'
  489. yield "__version__ = \"{}\"".format(ucdata.version)
  490. idx = -1
  491. for row in uts46_ranges(ucdata):
  492. idx += 1
  493. if idx % UTS46_SEGMENT_SIZE == 0:
  494. if idx != 0:
  495. yield " ]\n"
  496. yield "def _seg_{0}():\n return [".format(idx // UTS46_SEGMENT_SIZE)
  497. yield " {0},".format(row)
  498. yield " ]\n"
  499. yield "uts46data = tuple("
  500. yield " _seg_0()"
  501. for i in range(1, idx // UTS46_SEGMENT_SIZE + 1):
  502. yield " + _seg_{0}()".format(i)
  503. yield ")"
  504. def make_libdata(args, ucdata):
  505. dest_dir = args.dir or '.'
  506. target_filename = os.path.join(dest_dir, 'idnadata.py')
  507. with open(target_filename, 'wb') as target:
  508. for line in idna_libdata(ucdata):
  509. target.write((line + "\n").encode('utf-8'))
  510. target_filename = os.path.join(dest_dir, 'uts46data.py')
  511. with open(target_filename, 'wb') as target:
  512. for line in uts46_libdata(ucdata):
  513. target.write((line + "\n").encode('utf-8'))
  514. def arg_error(message, parser):
  515. parser.print_usage()
  516. print('{}: error: {}'.format(sys.argv[0], message))
  517. sys.exit(2)
  518. def main():
  519. parser = argparse.ArgumentParser(description='Determine IDNA code-point validity data')
  520. parser.add_argument('action', type=str, default='preferred',
  521. help='Task to perform (make-libdata, make-tables, <codepoint>)')
  522. parser.add_argument('--version', type=str, default='preferred',
  523. help='Unicode version to use (preferred, latest, <x.y.z>)')
  524. parser.add_argument('--source', type=str, default=None,
  525. help='Where to fetch Unicode data (file path)')
  526. parser.add_argument('--dir', type=str, default=None, help='Where to export the output')
  527. parser.add_argument('--cache', type=str, default=None, help='Where to cache Unicode data')
  528. parser.add_argument('--no-cache', action='store_true', help='Don\'t cache Unicode data')
  529. libdata = parser.add_argument_group('make-libdata', 'Make module data for Python IDNA library')
  530. tables = parser.add_argument_group('make-table', 'Make IANA-style reference table')
  531. codepoint = parser.add_argument_group('codepoint',
  532. 'Display related data for given codepoint (e.g. U+0061)')
  533. args = parser.parse_args()
  534. if args.version == 'preferred':
  535. target_version = PREFERRED_VERSION
  536. else:
  537. target_version = args.version
  538. if args.cache and args.no_cache:
  539. arg_error('I can\'t both --cache and --no-cache', parser)
  540. cache = args.cache or DEFAULT_CACHE_DIR
  541. if args.no_cache:
  542. cache = None
  543. ucdata = UnicodeData(target_version, cache, args)
  544. if args.action == 'make-table':
  545. make_table(args, ucdata)
  546. elif args.action == 'make-libdata':
  547. make_libdata(args, ucdata)
  548. else:
  549. result = re.match('^(?i)(U\+|)(?P<cp>[0-9A-F]{4,6})$', args.action)
  550. if result:
  551. codepoint = int(result.group('cp'), 16)
  552. diagnose_codepoint(codepoint, args, ucdata)
  553. sys.exit(0)
  554. arg_error('Don\'t recognize action or codepoint value', parser)
  555. if __name__ == '__main__':
  556. main()