tokenizer.js 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367
  1. /* ***** BEGIN LICENSE BLOCK *****
  2. * Distributed under the BSD license:
  3. *
  4. * Copyright (c) 2010, Ajax.org B.V.
  5. * All rights reserved.
  6. *
  7. * Redistribution and use in source and binary forms, with or without
  8. * modification, are permitted provided that the following conditions are met:
  9. * * Redistributions of source code must retain the above copyright
  10. * notice, this list of conditions and the following disclaimer.
  11. * * Redistributions in binary form must reproduce the above copyright
  12. * notice, this list of conditions and the following disclaimer in the
  13. * documentation and/or other materials provided with the distribution.
  14. * * Neither the name of Ajax.org B.V. nor the
  15. * names of its contributors may be used to endorse or promote products
  16. * derived from this software without specific prior written permission.
  17. *
  18. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  19. * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  21. * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
  22. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  23. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  25. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  27. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28. *
  29. * ***** END LICENSE BLOCK ***** */
  30. define(function(require, exports, module) {
  31. "use strict";
  32. var config = require("./config");
  33. // tokenizing lines longer than this makes editor very slow
  34. var MAX_TOKEN_COUNT = 2000;
  35. /**
  36. * This class takes a set of highlighting rules, and creates a tokenizer out of them. For more information, see [the wiki on extending highlighters](https://github.com/ajaxorg/ace/wiki/Creating-or-Extending-an-Edit-Mode#wiki-extendingTheHighlighter).
  37. * @class Tokenizer
  38. **/
  39. /**
  40. * Constructs a new tokenizer based on the given rules and flags.
  41. * @param {Object} rules The highlighting rules
  42. *
  43. * @constructor
  44. **/
  45. var Tokenizer = function(rules) {
  46. this.states = rules;
  47. this.regExps = {};
  48. this.matchMappings = {};
  49. for (var key in this.states) {
  50. var state = this.states[key];
  51. var ruleRegExps = [];
  52. var matchTotal = 0;
  53. var mapping = this.matchMappings[key] = {defaultToken: "text"};
  54. var flag = "g";
  55. var splitterRurles = [];
  56. for (var i = 0; i < state.length; i++) {
  57. var rule = state[i];
  58. if (rule.defaultToken)
  59. mapping.defaultToken = rule.defaultToken;
  60. if (rule.caseInsensitive)
  61. flag = "gi";
  62. if (rule.regex == null)
  63. continue;
  64. if (rule.regex instanceof RegExp)
  65. rule.regex = rule.regex.toString().slice(1, -1);
  66. // Count number of matching groups. 2 extra groups from the full match
  67. // And the catch-all on the end (used to force a match);
  68. var adjustedregex = rule.regex;
  69. var matchcount = new RegExp("(?:(" + adjustedregex + ")|(.))").exec("a").length - 2;
  70. if (Array.isArray(rule.token)) {
  71. if (rule.token.length == 1 || matchcount == 1) {
  72. rule.token = rule.token[0];
  73. } else if (matchcount - 1 != rule.token.length) {
  74. this.reportError("number of classes and regexp groups doesn't match", {
  75. rule: rule,
  76. groupCount: matchcount - 1
  77. });
  78. rule.token = rule.token[0];
  79. } else {
  80. rule.tokenArray = rule.token;
  81. rule.token = null;
  82. rule.onMatch = this.$arrayTokens;
  83. }
  84. } else if (typeof rule.token == "function" && !rule.onMatch) {
  85. if (matchcount > 1)
  86. rule.onMatch = this.$applyToken;
  87. else
  88. rule.onMatch = rule.token;
  89. }
  90. if (matchcount > 1) {
  91. if (/\\\d/.test(rule.regex)) {
  92. // Replace any backreferences and offset appropriately.
  93. adjustedregex = rule.regex.replace(/\\([0-9]+)/g, function(match, digit) {
  94. return "\\" + (parseInt(digit, 10) + matchTotal + 1);
  95. });
  96. } else {
  97. matchcount = 1;
  98. adjustedregex = this.removeCapturingGroups(rule.regex);
  99. }
  100. if (!rule.splitRegex && typeof rule.token != "string")
  101. splitterRurles.push(rule); // flag will be known only at the very end
  102. }
  103. mapping[matchTotal] = i;
  104. matchTotal += matchcount;
  105. ruleRegExps.push(adjustedregex);
  106. // makes property access faster
  107. if (!rule.onMatch)
  108. rule.onMatch = null;
  109. }
  110. if (!ruleRegExps.length) {
  111. mapping[0] = 0;
  112. ruleRegExps.push("$");
  113. }
  114. splitterRurles.forEach(function(rule) {
  115. rule.splitRegex = this.createSplitterRegexp(rule.regex, flag);
  116. }, this);
  117. this.regExps[key] = new RegExp("(" + ruleRegExps.join(")|(") + ")|($)", flag);
  118. }
  119. };
  120. (function() {
  121. this.$setMaxTokenCount = function(m) {
  122. MAX_TOKEN_COUNT = m | 0;
  123. };
  124. this.$applyToken = function(str) {
  125. var values = this.splitRegex.exec(str).slice(1);
  126. var types = this.token.apply(this, values);
  127. // required for compatibility with old modes
  128. if (typeof types === "string")
  129. return [{type: types, value: str}];
  130. var tokens = [];
  131. for (var i = 0, l = types.length; i < l; i++) {
  132. if (values[i])
  133. tokens[tokens.length] = {
  134. type: types[i],
  135. value: values[i]
  136. };
  137. }
  138. return tokens;
  139. },
  140. this.$arrayTokens = function(str) {
  141. if (!str)
  142. return [];
  143. var values = this.splitRegex.exec(str);
  144. if (!values)
  145. return "text";
  146. var tokens = [];
  147. var types = this.tokenArray;
  148. for (var i = 0, l = types.length; i < l; i++) {
  149. if (values[i + 1])
  150. tokens[tokens.length] = {
  151. type: types[i],
  152. value: values[i + 1]
  153. };
  154. }
  155. return tokens;
  156. };
  157. this.removeCapturingGroups = function(src) {
  158. var r = src.replace(
  159. /\[(?:\\.|[^\]])*?\]|\\.|\(\?[:=!]|(\()/g,
  160. function(x, y) {return y ? "(?:" : x;}
  161. );
  162. return r;
  163. };
  164. this.createSplitterRegexp = function(src, flag) {
  165. if (src.indexOf("(?=") != -1) {
  166. var stack = 0;
  167. var inChClass = false;
  168. var lastCapture = {};
  169. src.replace(/(\\.)|(\((?:\?[=!])?)|(\))|([\[\]])/g, function(
  170. m, esc, parenOpen, parenClose, square, index
  171. ) {
  172. if (inChClass) {
  173. inChClass = square != "]";
  174. } else if (square) {
  175. inChClass = true;
  176. } else if (parenClose) {
  177. if (stack == lastCapture.stack) {
  178. lastCapture.end = index+1;
  179. lastCapture.stack = -1;
  180. }
  181. stack--;
  182. } else if (parenOpen) {
  183. stack++;
  184. if (parenOpen.length != 1) {
  185. lastCapture.stack = stack
  186. lastCapture.start = index;
  187. }
  188. }
  189. return m;
  190. });
  191. if (lastCapture.end != null && /^\)*$/.test(src.substr(lastCapture.end)))
  192. src = src.substring(0, lastCapture.start) + src.substr(lastCapture.end);
  193. }
  194. // this is needed for regexps that can match in multiple ways
  195. if (src.charAt(0) != "^") src = "^" + src;
  196. if (src.charAt(src.length - 1) != "$") src += "$";
  197. return new RegExp(src, (flag||"").replace("g", ""));
  198. };
  199. /**
  200. * Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state.
  201. * @returns {Object}
  202. **/
  203. this.getLineTokens = function(line, startState) {
  204. if (startState && typeof startState != "string") {
  205. var stack = startState.slice(0);
  206. startState = stack[0];
  207. if (startState === "#tmp") {
  208. stack.shift()
  209. startState = stack.shift()
  210. }
  211. } else
  212. var stack = [];
  213. var currentState = startState || "start";
  214. var state = this.states[currentState];
  215. if (!state) {
  216. currentState = "start";
  217. state = this.states[currentState];
  218. }
  219. var mapping = this.matchMappings[currentState];
  220. var re = this.regExps[currentState];
  221. re.lastIndex = 0;
  222. var match, tokens = [];
  223. var lastIndex = 0;
  224. var matchAttempts = 0;
  225. var token = {type: null, value: ""};
  226. while (match = re.exec(line)) {
  227. var type = mapping.defaultToken;
  228. var rule = null;
  229. var value = match[0];
  230. var index = re.lastIndex;
  231. if (index - value.length > lastIndex) {
  232. var skipped = line.substring(lastIndex, index - value.length);
  233. if (token.type == type) {
  234. token.value += skipped;
  235. } else {
  236. if (token.type)
  237. tokens.push(token);
  238. token = {type: type, value: skipped};
  239. }
  240. }
  241. for (var i = 0; i < match.length-2; i++) {
  242. if (match[i + 1] === undefined)
  243. continue;
  244. rule = state[mapping[i]];
  245. if (rule.onMatch)
  246. type = rule.onMatch(value, currentState, stack);
  247. else
  248. type = rule.token;
  249. if (rule.next) {
  250. if (typeof rule.next == "string") {
  251. currentState = rule.next;
  252. } else {
  253. currentState = rule.next(currentState, stack);
  254. }
  255. state = this.states[currentState];
  256. if (!state) {
  257. this.reportError("state doesn't exist", currentState);
  258. currentState = "start";
  259. state = this.states[currentState];
  260. }
  261. mapping = this.matchMappings[currentState];
  262. lastIndex = index;
  263. re = this.regExps[currentState];
  264. re.lastIndex = index;
  265. }
  266. break;
  267. }
  268. if (value) {
  269. if (typeof type === "string") {
  270. if ((!rule || rule.merge !== false) && token.type === type) {
  271. token.value += value;
  272. } else {
  273. if (token.type)
  274. tokens.push(token);
  275. token = {type: type, value: value};
  276. }
  277. } else if (type) {
  278. if (token.type)
  279. tokens.push(token);
  280. token = {type: null, value: ""};
  281. for (var i = 0; i < type.length; i++)
  282. tokens.push(type[i]);
  283. }
  284. }
  285. if (lastIndex == line.length)
  286. break;
  287. lastIndex = index;
  288. if (matchAttempts++ > MAX_TOKEN_COUNT) {
  289. if (matchAttempts > 2 * line.length) {
  290. this.reportError("infinite loop with in ace tokenizer", {
  291. startState: startState,
  292. line: line
  293. });
  294. }
  295. // chrome doens't show contents of text nodes with very long text
  296. while (lastIndex < line.length) {
  297. if (token.type)
  298. tokens.push(token);
  299. token = {
  300. value: line.substring(lastIndex, lastIndex += 2000),
  301. type: "overflow"
  302. };
  303. }
  304. currentState = "start";
  305. stack = [];
  306. break;
  307. }
  308. }
  309. if (token.type)
  310. tokens.push(token);
  311. if (stack.length > 1) {
  312. if (stack[0] !== currentState)
  313. stack.unshift("#tmp", currentState);
  314. }
  315. return {
  316. tokens : tokens,
  317. state : stack.length ? stack : currentState
  318. };
  319. };
  320. this.reportError = config.reportError;
  321. }).call(Tokenizer.prototype);
  322. exports.Tokenizer = Tokenizer;
  323. });