tokenizer_dev.js 6.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183
  1. /* ***** BEGIN LICENSE BLOCK *****
  2. * Distributed under the BSD license:
  3. *
  4. * Copyright (c) 2010, Ajax.org B.V.
  5. * All rights reserved.
  6. *
  7. * Redistribution and use in source and binary forms, with or without
  8. * modification, are permitted provided that the following conditions are met:
  9. * * Redistributions of source code must retain the above copyright
  10. * notice, this list of conditions and the following disclaimer.
  11. * * Redistributions in binary form must reproduce the above copyright
  12. * notice, this list of conditions and the following disclaimer in the
  13. * documentation and/or other materials provided with the distribution.
  14. * * Neither the name of Ajax.org B.V. nor the
  15. * names of its contributors may be used to endorse or promote products
  16. * derived from this software without specific prior written permission.
  17. *
  18. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  19. * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  21. * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
  22. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  23. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  25. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  27. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28. *
  29. * ***** END LICENSE BLOCK ***** */
  30. define(function(require, exports, module) {
  31. var BaseTokenizer = require("./tokenizer").Tokenizer;
  32. // tokenizing lines longer than this makes editor very slow
  33. var MAX_TOKEN_COUNT = 100000;
  34. /*
  35. * version of Tokenizer with additional logging
  36. * and infinite loop checks
  37. * can be used for developing/testing new modes
  38. **/
  39. var Tokenizer = function(rules) {
  40. BaseTokenizer.call(this, rules);
  41. /**
  42. * Returns an object containing two properties: `tokens`, which contains all the tokens; and `state`, the current state.
  43. * @returns {Object}
  44. **/
  45. this.getLineTokens = function(line, startState) {
  46. if (startState && typeof startState != "string") {
  47. var stack = startState.slice(0);
  48. startState = stack[0];
  49. } else
  50. var stack = [];
  51. var currentState = startState || "start";
  52. var state = this.states[currentState];
  53. var mapping = this.matchMappings[currentState];
  54. var re = this.regExps[currentState];
  55. re.lastIndex = 0;
  56. var match, tokens = [];
  57. var lastIndex = 0;
  58. var stateTransitions = [];
  59. function onStateChange() {
  60. stateTransitions.push(startState+"@"+lastIndex);
  61. }
  62. function initState() {
  63. onStateChange();
  64. stateTransitions = [];
  65. onStateChange();
  66. }
  67. var token = {
  68. type: null,
  69. value: "",
  70. state: currentState
  71. };
  72. initState();
  73. var maxRecur = 100000;
  74. while (match = re.exec(line)) {
  75. var type = mapping.defaultToken;
  76. var rule = null;
  77. var value = match[0];
  78. var index = re.lastIndex;
  79. if (index - value.length > lastIndex) {
  80. var skipped = line.substring(lastIndex, index - value.length);
  81. if (token.type == type) {
  82. token.value += skipped;
  83. } else {
  84. if (token.type)
  85. tokens.push(token);
  86. token = {type: type, value: skipped};
  87. }
  88. }
  89. for (var i = 0; i < match.length-2; i++) {
  90. if (match[i + 1] === undefined)
  91. continue;
  92. if (!maxRecur--) {
  93. throw "infinite" + state[mapping[i]] + currentState
  94. }
  95. rule = state[mapping[i]];
  96. if (rule.onMatch)
  97. type = rule.onMatch(value, currentState, stack);
  98. else
  99. type = rule.token;
  100. if (rule.next) {
  101. if (typeof rule.next == "string")
  102. currentState = rule.next;
  103. else
  104. currentState = rule.next(currentState, stack);
  105. state = this.states[currentState];
  106. if (!state) {
  107. window.console && console.error && console.error(currentState, "doesn't exist");
  108. currentState = "start";
  109. state = this.states[currentState];
  110. }
  111. mapping = this.matchMappings[currentState];
  112. lastIndex = index;
  113. re = this.regExps[currentState];
  114. re.lastIndex = index;
  115. onStateChange();
  116. }
  117. break;
  118. }
  119. if (value) {
  120. if (typeof type == "string") {
  121. if ((!rule || rule.merge !== false) && token.type === type) {
  122. token.value += value;
  123. } else {
  124. if (token.type)
  125. tokens.push(token);
  126. token = {type: type, value: value};
  127. }
  128. } else {
  129. if (token.type)
  130. tokens.push(token);
  131. token = {type: null, value: ""};
  132. for (var i = 0; i < type.length; i++)
  133. tokens.push(type[i]);
  134. }
  135. }
  136. if (lastIndex == line.length)
  137. break;
  138. lastIndex = index;
  139. if (tokens.length > MAX_TOKEN_COUNT) {
  140. token.value += line.substr(lastIndex);
  141. currentState = "start"
  142. break;
  143. }
  144. }
  145. if (token.type)
  146. tokens.push(token);
  147. return {
  148. tokens : tokens,
  149. state : stack.length ? stack : currentState
  150. };
  151. };
  152. };
  153. Tokenizer.prototype = BaseTokenizer.prototype;
  154. exports.Tokenizer = Tokenizer;
  155. });