tokenizer_test.js 3.8 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697
  1. /* ***** BEGIN LICENSE BLOCK *****
  2. * Distributed under the BSD license:
  3. *
  4. * Copyright (c) 2010, Ajax.org B.V.
  5. * All rights reserved.
  6. *
  7. * Redistribution and use in source and binary forms, with or without
  8. * modification, are permitted provided that the following conditions are met:
  9. * * Redistributions of source code must retain the above copyright
  10. * notice, this list of conditions and the following disclaimer.
  11. * * Redistributions in binary form must reproduce the above copyright
  12. * notice, this list of conditions and the following disclaimer in the
  13. * documentation and/or other materials provided with the distribution.
  14. * * Neither the name of Ajax.org B.V. nor the
  15. * names of its contributors may be used to endorse or promote products
  16. * derived from this software without specific prior written permission.
  17. *
  18. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  19. * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  21. * DISCLAIMED. IN NO EVENT SHALL AJAX.ORG B.V. BE LIABLE FOR ANY
  22. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  23. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  24. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  25. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  27. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28. *
  29. * ***** END LICENSE BLOCK ***** */
  30. if (typeof process !== "undefined") {
  31. require("amd-loader");
  32. }
  33. define(function(require, exports, module) {
  34. "use strict";
  35. var Tokenizer = require("./tokenizer").Tokenizer;
  36. var assert = require("./test/assertions");
  37. module.exports = {
  38. "test: createSplitterRegexp" : function() {
  39. var t = new Tokenizer({});
  40. var re = t.createSplitterRegexp("(a)(b)(?=[x)(])");
  41. assert.equal(re.source, "^(a)(b)$");
  42. var re = t.createSplitterRegexp("xc(?=([x)(]))");
  43. assert.equal(re.source, "^xc$");
  44. var re = t.createSplitterRegexp("(xc(?=([x)(])))");
  45. assert.equal(re.source, "^(xc)$");
  46. var re = t.createSplitterRegexp("(?=r)[(?=)](?=([x)(]))");
  47. assert.equal(re.source, "^(?=r)[(?=)]$");
  48. var re = t.createSplitterRegexp("(?=r)[(?=)](\\?=t)");
  49. assert.equal(re.source, "^(?=r)[(?=)](\\?=t)$");
  50. var re = t.createSplitterRegexp("[(?=)](\\?=t)");
  51. assert.equal(re.source, "^[(?=)](\\?=t)$");
  52. },
  53. "test: removeCapturingGroups" : function() {
  54. var t = new Tokenizer({});
  55. var re = t.removeCapturingGroups("(ax(by))[()]");
  56. assert.equal(re, "(?:ax(?:by))[()]");
  57. },
  58. "test: broken highlight rules": function() {
  59. var t = new Tokenizer({
  60. start: [{
  61. token: 's',
  62. regex: '&&&|^^^'
  63. }, {
  64. defaultToken: "def"
  65. }],
  66. state1: [{
  67. token: 'x',
  68. regex: /\b([\w]*)(\s*)((?::)?)/
  69. }]
  70. });
  71. var errorReports = 0;
  72. t.reportError = function() { errorReports++; };
  73. var tokens = t.getLineTokens("x|", "start");
  74. assert.deepEqual(tokens, {
  75. tokens: [{value: 'x|', type: 'overflow'}],
  76. state: 'start'
  77. });
  78. var tokens = t.getLineTokens("x|", "state1");
  79. assert.deepEqual(tokens, {
  80. tokens: [{value: 'x', type: 'x'}, {value: '|', type: 'overflow'}],
  81. state: 'start'
  82. });
  83. assert.equal(errorReports, 2);
  84. },
  85. };
  86. });
  87. if (typeof module !== "undefined" && module === require.main) {
  88. require("asyncjs").test.testcase(module.exports).exec()
  89. }