background_tokenizer.js 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. /* ***** BEGIN LICENSE BLOCK *****
  2. * Distributed under the BSD license:
  3. *
  4. * Copyright (c) 2010, Ajax.org B.V.
  5. * All rights reserved.
  6. *
  7. * Redistribution and use in source and binary forms, with or without
  8. * modification, are permitted provided that the following conditions are met:
  9. * * Redistributions of source code must retain the above copyright
  10. * notice, this list of conditions and the following disclaimer.
  11. * * Redistributions in binary form must reproduce the above copyright
  12. * notice, this list of conditions and the following disclaimer in the
  13. * documentation and/or other materials provided with the distribution.
  14. * * Neither the name of Ajax.org B.V. nor the
  15. * names of its contributors may be used to endorse or promote products
  16. * derived from this software without specific prior written permission.
  17. *
  18. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  19. * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  21. * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
  22. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  23. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  25. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  27. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28. *
  29. * ***** END LICENSE BLOCK ***** */
  30. define(function(require, exports, module) {
  31. "use strict";
  32. var oop = require("./lib/oop");
  33. var EventEmitter = require("./lib/event_emitter").EventEmitter;
  34. /**
  35. * Tokenizes the current [[Document `Document`]] in the background, and caches the tokenized rows for future use.
  36. *
  37. * If a certain row is changed, everything below that row is re-tokenized.
  38. *
  39. * @class BackgroundTokenizer
  40. **/
  41. /**
  42. * Creates a new `BackgroundTokenizer` object.
  43. * @param {Tokenizer} tokenizer The tokenizer to use
  44. * @param {Editor} editor The editor to associate with
  45. *
  46. * @constructor
  47. **/
  48. var BackgroundTokenizer = function(tokenizer, editor) {
  49. this.running = false;
  50. this.lines = [];
  51. this.states = [];
  52. this.currentLine = 0;
  53. this.tokenizer = tokenizer;
  54. var self = this;
  55. this.$worker = function() {
  56. if (!self.running) { return; }
  57. var workerStart = new Date();
  58. var currentLine = self.currentLine;
  59. var endLine = -1;
  60. var doc = self.doc;
  61. var startLine = currentLine;
  62. while (self.lines[currentLine])
  63. currentLine++;
  64. var len = doc.getLength();
  65. var processedLines = 0;
  66. self.running = false;
  67. while (currentLine < len) {
  68. self.$tokenizeRow(currentLine);
  69. endLine = currentLine;
  70. do {
  71. currentLine++;
  72. } while (self.lines[currentLine]);
  73. // only check every 5 lines
  74. processedLines ++;
  75. if ((processedLines % 5 === 0) && (new Date() - workerStart) > 20) {
  76. self.running = setTimeout(self.$worker, 20);
  77. break;
  78. }
  79. }
  80. self.currentLine = currentLine;
  81. if (startLine <= endLine)
  82. self.fireUpdateEvent(startLine, endLine);
  83. };
  84. };
  85. (function(){
  86. oop.implement(this, EventEmitter);
  87. /**
  88. * Sets a new tokenizer for this object.
  89. *
  90. * @param {Tokenizer} tokenizer The new tokenizer to use
  91. *
  92. **/
  93. this.setTokenizer = function(tokenizer) {
  94. this.tokenizer = tokenizer;
  95. this.lines = [];
  96. this.states = [];
  97. this.start(0);
  98. };
  99. /**
  100. * Sets a new document to associate with this object.
  101. * @param {Document} doc The new document to associate with
  102. **/
  103. this.setDocument = function(doc) {
  104. this.doc = doc;
  105. this.lines = [];
  106. this.states = [];
  107. this.stop();
  108. };
  109. /**
  110. * Fires whenever the background tokeniziers between a range of rows are going to be updated.
  111. *
  112. * @event update
  113. * @param {Object} e An object containing two properties, `first` and `last`, which indicate the rows of the region being updated.
  114. *
  115. **/
  116. /**
  117. * Emits the `'update'` event. `firstRow` and `lastRow` are used to define the boundaries of the region to be updated.
  118. * @param {Number} firstRow The starting row region
  119. * @param {Number} lastRow The final row region
  120. *
  121. **/
  122. this.fireUpdateEvent = function(firstRow, lastRow) {
  123. var data = {
  124. first: firstRow,
  125. last: lastRow
  126. };
  127. this._signal("update", {data: data});
  128. };
  129. /**
  130. * Starts tokenizing at the row indicated.
  131. *
  132. * @param {Number} startRow The row to start at
  133. *
  134. **/
  135. this.start = function(startRow) {
  136. this.currentLine = Math.min(startRow || 0, this.currentLine, this.doc.getLength());
  137. // remove all cached items below this line
  138. this.lines.splice(this.currentLine, this.lines.length);
  139. this.states.splice(this.currentLine, this.states.length);
  140. this.stop();
  141. // pretty long delay to prevent the tokenizer from interfering with the user
  142. this.running = setTimeout(this.$worker, 700);
  143. };
  144. this.scheduleStart = function() {
  145. if (!this.running)
  146. this.running = setTimeout(this.$worker, 700);
  147. }
  148. this.$updateOnChange = function(delta) {
  149. var startRow = delta.start.row;
  150. var len = delta.end.row - startRow;
  151. if (len === 0) {
  152. this.lines[startRow] = null;
  153. } else if (delta.action == "remove") {
  154. this.lines.splice(startRow, len + 1, null);
  155. this.states.splice(startRow, len + 1, null);
  156. } else {
  157. var args = Array(len + 1);
  158. args.unshift(startRow, 1);
  159. this.lines.splice.apply(this.lines, args);
  160. this.states.splice.apply(this.states, args);
  161. }
  162. this.currentLine = Math.min(startRow, this.currentLine, this.doc.getLength());
  163. this.stop();
  164. };
  165. /**
  166. * Stops tokenizing.
  167. *
  168. **/
  169. this.stop = function() {
  170. if (this.running)
  171. clearTimeout(this.running);
  172. this.running = false;
  173. };
  174. /**
  175. * Gives list of tokens of the row. (tokens are cached)
  176. *
  177. * @param {Number} row The row to get tokens at
  178. *
  179. *
  180. *
  181. **/
  182. this.getTokens = function(row) {
  183. return this.lines[row] || this.$tokenizeRow(row);
  184. };
  185. /**
  186. * [Returns the state of tokenization at the end of a row.]{: #BackgroundTokenizer.getState}
  187. *
  188. * @param {Number} row The row to get state at
  189. **/
  190. this.getState = function(row) {
  191. if (this.currentLine == row)
  192. this.$tokenizeRow(row);
  193. return this.states[row] || "start";
  194. };
  195. this.$tokenizeRow = function(row) {
  196. var line = this.doc.getLine(row);
  197. var state = this.states[row - 1];
  198. var data = this.tokenizer.getLineTokens(line, state, row);
  199. if (this.states[row] + "" !== data.state + "") {
  200. this.states[row] = data.state;
  201. this.lines[row + 1] = null;
  202. if (this.currentLine > row + 1)
  203. this.currentLine = row + 1;
  204. } else if (this.currentLine == row) {
  205. this.currentLine = row + 1;
  206. }
  207. return this.lines[row] = data.tokens;
  208. };
  209. }).call(BackgroundTokenizer.prototype);
  210. exports.BackgroundTokenizer = BackgroundTokenizer;
  211. });