localedata.py 7.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238
  1. # -*- coding: utf-8 -*-
  2. """
  3. babel.localedata
  4. ~~~~~~~~~~~~~~~~
  5. Low-level locale data access.
  6. :note: The `Locale` class, which uses this module under the hood, provides a
  7. more convenient interface for accessing the locale data.
  8. :copyright: (c) 2013 by the Babel Team.
  9. :license: BSD, see LICENSE for more details.
  10. """
  11. import os
  12. import threading
  13. from collections import MutableMapping
  14. from itertools import chain
  15. import sys
  16. from babel._compat import pickle, string_types
  17. def get_base_dir():
  18. if getattr(sys, 'frozen', False) and getattr(sys, '_MEIPASS', None):
  19. # we are running in a |PyInstaller| bundle
  20. basedir = sys._MEIPASS
  21. else:
  22. # we are running in a normal Python environment
  23. basedir = os.path.dirname(__file__)
  24. return basedir
  25. _cache = {}
  26. _cache_lock = threading.RLock()
  27. _dirname = os.path.join(get_base_dir(), 'locale-data')
  28. def normalize_locale(name):
  29. """Normalize a locale ID by stripping spaces and apply proper casing.
  30. Returns the normalized locale ID string or `None` if the ID is not
  31. recognized.
  32. """
  33. if not name or not isinstance(name, string_types):
  34. return None
  35. name = name.strip().lower()
  36. for locale_id in chain.from_iterable([_cache, locale_identifiers()]):
  37. if name == locale_id.lower():
  38. return locale_id
  39. def exists(name):
  40. """Check whether locale data is available for the given locale.
  41. Returns `True` if it exists, `False` otherwise.
  42. :param name: the locale identifier string
  43. """
  44. if not name or not isinstance(name, string_types):
  45. return False
  46. if name in _cache:
  47. return True
  48. file_found = os.path.exists(os.path.join(_dirname, '%s.dat' % name))
  49. return True if file_found else bool(normalize_locale(name))
  50. def locale_identifiers():
  51. """Return a list of all locale identifiers for which locale data is
  52. available.
  53. .. versionadded:: 0.8.1
  54. :return: a list of locale identifiers (strings)
  55. """
  56. return [stem for stem, extension in [
  57. os.path.splitext(filename) for filename in os.listdir(_dirname)
  58. ] if extension == '.dat' and stem != 'root']
  59. def load(name, merge_inherited=True):
  60. """Load the locale data for the given locale.
  61. The locale data is a dictionary that contains much of the data defined by
  62. the Common Locale Data Repository (CLDR). This data is stored as a
  63. collection of pickle files inside the ``babel`` package.
  64. >>> d = load('en_US')
  65. >>> d['languages']['sv']
  66. u'Swedish'
  67. Note that the results are cached, and subsequent requests for the same
  68. locale return the same dictionary:
  69. >>> d1 = load('en_US')
  70. >>> d2 = load('en_US')
  71. >>> d1 is d2
  72. True
  73. :param name: the locale identifier string (or "root")
  74. :param merge_inherited: whether the inherited data should be merged into
  75. the data of the requested locale
  76. :raise `IOError`: if no locale data file is found for the given locale
  77. identifer, or one of the locales it inherits from
  78. """
  79. _cache_lock.acquire()
  80. try:
  81. data = _cache.get(name)
  82. if not data:
  83. # Load inherited data
  84. if name == 'root' or not merge_inherited:
  85. data = {}
  86. else:
  87. from babel.core import get_global
  88. parent = get_global('parent_exceptions').get(name)
  89. if not parent:
  90. parts = name.split('_')
  91. if len(parts) == 1:
  92. parent = 'root'
  93. else:
  94. parent = '_'.join(parts[:-1])
  95. data = load(parent).copy()
  96. filename = os.path.join(_dirname, '%s.dat' % name)
  97. with open(filename, 'rb') as fileobj:
  98. if name != 'root' and merge_inherited:
  99. merge(data, pickle.load(fileobj))
  100. else:
  101. data = pickle.load(fileobj)
  102. _cache[name] = data
  103. return data
  104. finally:
  105. _cache_lock.release()
  106. def merge(dict1, dict2):
  107. """Merge the data from `dict2` into the `dict1` dictionary, making copies
  108. of nested dictionaries.
  109. >>> d = {1: 'foo', 3: 'baz'}
  110. >>> merge(d, {1: 'Foo', 2: 'Bar'})
  111. >>> sorted(d.items())
  112. [(1, 'Foo'), (2, 'Bar'), (3, 'baz')]
  113. :param dict1: the dictionary to merge into
  114. :param dict2: the dictionary containing the data that should be merged
  115. """
  116. for key, val2 in dict2.items():
  117. if val2 is not None:
  118. val1 = dict1.get(key)
  119. if isinstance(val2, dict):
  120. if val1 is None:
  121. val1 = {}
  122. if isinstance(val1, Alias):
  123. val1 = (val1, val2)
  124. elif isinstance(val1, tuple):
  125. alias, others = val1
  126. others = others.copy()
  127. merge(others, val2)
  128. val1 = (alias, others)
  129. else:
  130. val1 = val1.copy()
  131. merge(val1, val2)
  132. else:
  133. val1 = val2
  134. dict1[key] = val1
  135. class Alias(object):
  136. """Representation of an alias in the locale data.
  137. An alias is a value that refers to some other part of the locale data,
  138. as specified by the `keys`.
  139. """
  140. def __init__(self, keys):
  141. self.keys = tuple(keys)
  142. def __repr__(self):
  143. return '<%s %r>' % (type(self).__name__, self.keys)
  144. def resolve(self, data):
  145. """Resolve the alias based on the given data.
  146. This is done recursively, so if one alias resolves to a second alias,
  147. that second alias will also be resolved.
  148. :param data: the locale data
  149. :type data: `dict`
  150. """
  151. base = data
  152. for key in self.keys:
  153. data = data[key]
  154. if isinstance(data, Alias):
  155. data = data.resolve(base)
  156. elif isinstance(data, tuple):
  157. alias, others = data
  158. data = alias.resolve(base)
  159. return data
  160. class LocaleDataDict(MutableMapping):
  161. """Dictionary wrapper that automatically resolves aliases to the actual
  162. values.
  163. """
  164. def __init__(self, data, base=None):
  165. self._data = data
  166. if base is None:
  167. base = data
  168. self.base = base
  169. def __len__(self):
  170. return len(self._data)
  171. def __iter__(self):
  172. return iter(self._data)
  173. def __getitem__(self, key):
  174. orig = val = self._data[key]
  175. if isinstance(val, Alias): # resolve an alias
  176. val = val.resolve(self.base)
  177. if isinstance(val, tuple): # Merge a partial dict with an alias
  178. alias, others = val
  179. val = alias.resolve(self.base).copy()
  180. merge(val, others)
  181. if type(val) is dict: # Return a nested alias-resolving dict
  182. val = LocaleDataDict(val, base=self.base)
  183. if val is not orig:
  184. self._data[key] = val
  185. return val
  186. def __setitem__(self, key, value):
  187. self._data[key] = value
  188. def __delitem__(self, key):
  189. del self._data[key]
  190. def copy(self):
  191. return LocaleDataDict(self._data.copy(), base=self.base)