token.py 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198
  1. # -*- coding: utf-8 -*-
  2. """
  3. pygments.token
  4. ~~~~~~~~~~~~~~
  5. Basic token types and the standard tokens.
  6. :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
  7. :license: BSD, see LICENSE for details.
  8. """
  9. class _TokenType(tuple):
  10. parent = None
  11. def split(self):
  12. buf = []
  13. node = self
  14. while node is not None:
  15. buf.append(node)
  16. node = node.parent
  17. buf.reverse()
  18. return buf
  19. def __init__(self, *args):
  20. # no need to call super.__init__
  21. self.subtypes = set()
  22. def __contains__(self, val):
  23. return self is val or (
  24. type(val) is self.__class__ and
  25. val[:len(self)] == self
  26. )
  27. def __getattr__(self, val):
  28. if not val or not val[0].isupper():
  29. return tuple.__getattribute__(self, val)
  30. new = _TokenType(self + (val,))
  31. setattr(self, val, new)
  32. self.subtypes.add(new)
  33. new.parent = self
  34. return new
  35. def __hash__(self):
  36. return hash(tuple(self))
  37. def __repr__(self):
  38. return 'Token' + (self and '.' or '') + '.'.join(self)
  39. Token = _TokenType()
  40. # Special token types
  41. Text = Token.Text
  42. Whitespace = Text.Whitespace
  43. Error = Token.Error
  44. # Text that doesn't belong to this lexer (e.g. HTML in PHP)
  45. Other = Token.Other
  46. # Common token types for source code
  47. Keyword = Token.Keyword
  48. Name = Token.Name
  49. Literal = Token.Literal
  50. String = Literal.String
  51. Number = Literal.Number
  52. Punctuation = Token.Punctuation
  53. Operator = Token.Operator
  54. Comment = Token.Comment
  55. # Generic types for non-source code
  56. Generic = Token.Generic
  57. # String and some others are not direct childs of Token.
  58. # alias them:
  59. Token.Token = Token
  60. Token.String = String
  61. Token.Number = Number
  62. def is_token_subtype(ttype, other):
  63. """
  64. Return True if ``ttype`` is a subtype of ``other``.
  65. exists for backwards compatibility. use ``ttype in other`` now.
  66. """
  67. return ttype in other
  68. def string_to_tokentype(s):
  69. """
  70. Convert a string into a token type::
  71. >>> string_to_token('String.Double')
  72. Token.Literal.String.Double
  73. >>> string_to_token('Token.Literal.Number')
  74. Token.Literal.Number
  75. >>> string_to_token('')
  76. Token
  77. Tokens that are already tokens are returned unchanged:
  78. >>> string_to_token(String)
  79. Token.Literal.String
  80. """
  81. if isinstance(s, _TokenType):
  82. return s
  83. if not s:
  84. return Token
  85. node = Token
  86. for item in s.split('.'):
  87. node = getattr(node, item)
  88. return node
  89. # Map standard token types to short names, used in CSS class naming.
  90. # If you add a new item, please be sure to run this file to perform
  91. # a consistency check for duplicate values.
  92. STANDARD_TYPES = {
  93. Token: '',
  94. Text: '',
  95. Whitespace: 'w',
  96. Error: 'err',
  97. Other: 'x',
  98. Keyword: 'k',
  99. Keyword.Constant: 'kc',
  100. Keyword.Declaration: 'kd',
  101. Keyword.Namespace: 'kn',
  102. Keyword.Pseudo: 'kp',
  103. Keyword.Reserved: 'kr',
  104. Keyword.Type: 'kt',
  105. Name: 'n',
  106. Name.Attribute: 'na',
  107. Name.Builtin: 'nb',
  108. Name.Builtin.Pseudo: 'bp',
  109. Name.Class: 'nc',
  110. Name.Constant: 'no',
  111. Name.Decorator: 'nd',
  112. Name.Entity: 'ni',
  113. Name.Exception: 'ne',
  114. Name.Function: 'nf',
  115. Name.Property: 'py',
  116. Name.Label: 'nl',
  117. Name.Namespace: 'nn',
  118. Name.Other: 'nx',
  119. Name.Tag: 'nt',
  120. Name.Variable: 'nv',
  121. Name.Variable.Class: 'vc',
  122. Name.Variable.Global: 'vg',
  123. Name.Variable.Instance: 'vi',
  124. Literal: 'l',
  125. Literal.Date: 'ld',
  126. String: 's',
  127. String.Backtick: 'sb',
  128. String.Char: 'sc',
  129. String.Doc: 'sd',
  130. String.Double: 's2',
  131. String.Escape: 'se',
  132. String.Heredoc: 'sh',
  133. String.Interpol: 'si',
  134. String.Other: 'sx',
  135. String.Regex: 'sr',
  136. String.Single: 's1',
  137. String.Symbol: 'ss',
  138. Number: 'm',
  139. Number.Float: 'mf',
  140. Number.Hex: 'mh',
  141. Number.Integer: 'mi',
  142. Number.Integer.Long: 'il',
  143. Number.Oct: 'mo',
  144. Operator: 'o',
  145. Operator.Word: 'ow',
  146. Punctuation: 'p',
  147. Comment: 'c',
  148. Comment.Multiline: 'cm',
  149. Comment.Preproc: 'cp',
  150. Comment.Single: 'c1',
  151. Comment.Special: 'cs',
  152. Generic: 'g',
  153. Generic.Deleted: 'gd',
  154. Generic.Emph: 'ge',
  155. Generic.Error: 'gr',
  156. Generic.Heading: 'gh',
  157. Generic.Inserted: 'gi',
  158. Generic.Output: 'go',
  159. Generic.Prompt: 'gp',
  160. Generic.Strong: 'gs',
  161. Generic.Subheading: 'gu',
  162. Generic.Traceback: 'gt',
  163. }