regexp_tokenizer.js 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114
  1. /***** regexp tokenizer */
  2. require("amd-loader");
  3. var lib = require("./lib");
  4. var Tokenizer = require(lib.AceLib+ "ace/tokenizer").Tokenizer;
  5. var Tokenizer = require(lib.AceLib + "ace/tokenizer_dev").Tokenizer; // todo can't use tokenizer because of max token count
  6. var TextHighlightRules = require(lib.AceLib + "ace/mode/text_highlight_rules").TextHighlightRules;
  7. var r = new TextHighlightRules()
  8. r.$rules = {
  9. start: [
  10. {token: "anchor", regex: /[\^\$]|\\[bBAZzG]/, merge:false},
  11. {token: "backRef", regex: /\\([1-9]|k(<\w+\b[+-]?\d>|'\w+\b[+-]?\d'))/, merge:false},
  12. {include: "charTypes", merge:false},
  13. {token: "charclass", regex: /\[\^?/, push: "charclass", merge:false},
  14. {token: "alternation", regex: /\|/, merge:false},
  15. {include: "quantifiers", merge:false},
  16. {include: "groups", merge:false},
  17. {include: "xGroup", merge:true}
  18. ],
  19. charTypes: [
  20. {token: "char", regex: /\\([tvnrbfae]|[0-8]{1,3}|x[\dA-Fa-f]{2}|x7[\dA-Fa-f]{7})/, merge:false}, // todo \cx
  21. {token: "charType", regex: /\.|\\[wWsSdDhH]/, merge:false},
  22. {token: "charProperty", regex: /\\p{\w+}/, merge:false},
  23. {token: "char", regex: /\\./, merge:false},
  24. ],
  25. quantifiers: [
  26. {token: "quantifier", regex: /([?*+]|{\d+\b,?\d*}|{,\d+})[?+]?/, merge:false}
  27. ],
  28. charclass: [
  29. {include: "charTypes", merge:false},
  30. {token: "charclass.start", regex: /\[\^?/, push: "charclass", merge:false},
  31. {token: "charclass.end", regex: /\]/, next: "pop", merge:false}
  32. ],
  33. groups: [
  34. {token: "group", regex: /[(]([?](#|[imx\-]+:?|:|=|!|<=|<!|>|<\w+>|'\w+'|))?|[)]/,
  35. onMatch: function(val, state, stack) {
  36. if (!stack.groupNumber)
  37. stack.groupNumber = 1;
  38. var isStart = val !== ")";
  39. var t = {depth:0,type: isStart ? "group.start" : "group.end", value: val};
  40. t.groupType = val[2];
  41. if (val == "(") {
  42. t.number = stack.groupNumber++;
  43. t.isGroup = true
  44. } else if (t.groupType == "'" || (t.groupType == "<" && val.slice(-1) == ">")) {
  45. t.name = val.slice(2, -1)
  46. t.isGroup = true
  47. } else if (t.groupType == ":") {
  48. t.isGroup = true
  49. }
  50. if (t.groupType && val.indexOf("x") != -1) {
  51. var minus = val.indexOf("-");
  52. if (minus == -1 || minus > val.indexOf("x"))
  53. stack.xGroup = t;
  54. else
  55. stack.xGroup = null;
  56. } else if (!isStart && stack.xGroup && stack.xGroup == stack[0]) {
  57. if (stack.xGroup.value.slice(-1) == ":")
  58. stack.xGroup = null;
  59. }
  60. if (isStart) {
  61. if (stack.groupDepth) {
  62. stack[0].hasChildren = true
  63. }
  64. stack.groupDepth = (stack.groupDepth||0)+1;
  65. stack.unshift(t)
  66. } else {
  67. stack.groupDepth --;
  68. t.start = stack.shift(t)
  69. t.start.end = t
  70. }
  71. return [t]
  72. }, merge:false
  73. }
  74. ],
  75. xGroup: [
  76. {token: "text", regex:/\s+/, onMatch: function(val, state, stack) {
  77. return stack.xGroup ? [] : "text"
  78. }, merge: true},
  79. {token: "text", regex: /#/, onMatch: function(val, state, stack) {
  80. if (stack.xGroup) {
  81. this.next = "comment";
  82. stack.unshift(state);
  83. return [];
  84. }
  85. this.next = "";
  86. return "text";
  87. }, merge: true}
  88. ],
  89. comment: [{
  90. regex: "[^\n\r]*|^", token: "", onMatch: function(val, state, stack) {
  91. this.next = stack.shift();
  92. return [];
  93. }
  94. }]
  95. }
  96. r.normalizeRules()
  97. var tmReTokenizer = new Tokenizer(r.getRules());
  98. function tokenize(str) {
  99. return tmReTokenizer.getLineTokens(str).tokens;
  100. }
  101. function toStr(tokens) { return tokens.map(function(x){return x.value}).join("")}
  102. exports.tokenize = tokenize;
  103. exports.toStr = toStr;
  104. exports.tmReTokenizer = tmReTokenizer;