tmlanguage.js 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697
  1. require("amd-loader");
  2. var fs = require("fs");
  3. var util = require("util");
  4. var lib = require("./lib");
  5. var pathlib = require("path");
  6. var parseLanguage = lib.parsePlist;
  7. var tk = require("./regexp_tokenizer");
  8. var tokenize = tk.tokenize;
  9. var toStr = tk.toStr;
  10. function last(array) {return array[array.length - 1]}
  11. function convertHexEscape(tokens) {
  12. var inChClass = false;
  13. tokens.forEach(function(t) {
  14. if (t.type == "charclass")
  15. inChClass = true;
  16. else if (t.type == "charclass.end")
  17. inChClass = false;
  18. else if (t.type == "charType"){
  19. if (t.value == "\\h") {
  20. t.type = "text";
  21. t.value = inChClass ? "\\da-fA-F" : "[\\da-fA-F]";
  22. }
  23. else if (t.value == "\\H") {
  24. if (inChClass) {
  25. console.warn("can't convert \\H in charclass");
  26. return;
  27. }
  28. t.type = "text";
  29. t.value = "[^\\da-fA-F]";
  30. }
  31. }
  32. });
  33. return tokens;
  34. }
  35. function convertNewLinesTo$(str) {
  36. var tokens = tokenize(str);
  37. for (var i = 0; i < tokens.length; i++) {
  38. var t= tokens[i];
  39. if (t.type == "char" && t.value == "\\n") {
  40. var p = tokens[i + 1] || {};
  41. if (p.type != "quantifier") {
  42. t.value = "$";
  43. while (p.value == "\\n" || p.type == "quantifier") {
  44. p.value = "";
  45. p = tokens[++i + 1] || {};
  46. }
  47. } else if (/\?|\*|{,|{0,/.test(p.value)) {
  48. t.value = p.value = "";
  49. } else
  50. p.value = "";
  51. }
  52. }
  53. return toStr(tokens).replace(/[$]+/g, "$");
  54. }
  55. function convertCharacterTypes(str) {
  56. var tokens = tokenize(str);
  57. tokens = convertHexEscape(tokens);
  58. var warn = false;
  59. tokens.forEach(function(t){
  60. if (t.type == "quantifier") {
  61. var val = t.value;
  62. if (val.slice(-1) == "+" && val.length > 1) {
  63. t.value = val.slice(0, -1);
  64. warn = val;
  65. }
  66. }
  67. });
  68. if (warn)
  69. console.log("converted possesive quantifier " + warn + " to *");
  70. return toStr(tokens);
  71. }
  72. function removeInlineFlags(str, rule) {
  73. var tokens = tokenize(str);
  74. var caseInsensitive = false;
  75. tokens.forEach(function(t, i) {
  76. if (t.type == "group.start" && /[imsx]/.test(t.value)) {
  77. if (/i/.test(t.value))
  78. caseInsensitive = true;
  79. t.value = t.value.replace(/[imsx\-]/g, "");
  80. var next = tokens[i + 1];
  81. if (next && next.type == "group.end") {
  82. t.value = next.value = "";
  83. }
  84. }
  85. });
  86. if (caseInsensitive && rule)
  87. rule.caseInsensitive = true;
  88. return toStr(tokens);
  89. }
  90. function convertToNonCapturingGroups(str) {
  91. var tokens = tokenize(str);
  92. tokens.forEach(function(t, i) {
  93. if (t.type == "group.start" && t.value == "(")
  94. t.value += "?:";
  95. });
  96. return toStr(tokens);
  97. }
  98. function simplifyNonCapturingGroups(str) {
  99. var tokens = tokenize(str);
  100. var t = tokens[0] || {};
  101. if (t.type == "group.start" && t.value == "(?:"
  102. && t.end == last(tokens)) {
  103. t.value = t.end.value = "";
  104. }
  105. var i = 0;
  106. function iter(f) {
  107. for (i = 0; i < tokens.length; i++)
  108. f(tokens[i]);
  109. }
  110. function iterGroup(end, f) {
  111. for (var i1 = i + 1; i1 < tokens.length; i1++) {
  112. var t = tokens[i1];
  113. if (t == end)
  114. break;
  115. var index = f && f(t);
  116. if (index > i1)
  117. i1 = index;
  118. }
  119. return i1;
  120. }
  121. iter(function (t) {
  122. if (t.type == "group.start" && t.value == "(?:") {
  123. if (!t.end)
  124. return console.error("malformed regex: " + str);
  125. var canRemove = true;
  126. var next = tokens[tokens.indexOf(t.end, i) + 1];
  127. if (next && next.type == "quantifier")
  128. return;
  129. iterGroup(t.end, function(t) {
  130. if (t.type == "alternation")
  131. canRemove = false;
  132. else if (t.type == "group.start" && t.end)
  133. return iterGroup(t.end);
  134. });
  135. if (canRemove)
  136. t.value = t.end.value = "";
  137. }
  138. });
  139. return toStr(tokens);
  140. }
  141. function removeLookBehinds(str) {
  142. var tokens = tokenize(str);
  143. var toRemove = null;
  144. tokens.forEach(function(t, i) {
  145. if (!toRemove && t.type == "group.start" && /</.test(t.value)) {
  146. toRemove = t.end;
  147. toRemove.content = [];
  148. }
  149. if (toRemove) {
  150. toRemove.content.push(t.value);
  151. t.value = "";
  152. }
  153. if (t == toRemove) {
  154. var c = toRemove.content.slice(1, -1).join("");
  155. if (/\^/.test(c))
  156. toRemove.value = "(?:" + c +")";
  157. toRemove = null;
  158. }
  159. });
  160. return toStr(tokens);
  161. }
  162. function convertBeginEndBackrefs(rule) {
  163. if (!/\\\d/.test(rule.end))
  164. return;
  165. var startTokens = tokenize(rule.begin);
  166. var endTokens = tokenize(rule.end);
  167. var groups = {};
  168. startTokens.forEach(function(t, i) {
  169. if (t.number && t.end && t.type == "group.start") {
  170. var endIndex = startTokens.indexOf(t.end, i + 1);
  171. var content = startTokens.slice(i+1, endIndex);
  172. groups[t.number] = toStr(content);
  173. }
  174. });
  175. endTokens.forEach(function(t) {
  176. if (t.type == "backRef") {
  177. var num = t.value.substr(1);
  178. if (groups[num])
  179. t.value = "(?:" + groups[num] + ")";
  180. }
  181. });
  182. rule.end = toStr(endTokens);
  183. console.warn("Begin-End-Backreference is detected", rule);
  184. }
  185. function checkForNamedCaptures(str) {
  186. var tokens = tokenize(str);
  187. tokens.forEach(function(t) {
  188. if (t.type == "group.start" && t.name)
  189. console.warn("named capture not implemented", str);
  190. if (t.type == "backRef")
  191. console.warn("backRef not implemented ", str);
  192. });
  193. }
  194. function fixGroups(captures, defaultName, regex) {
  195. var tokens = tokenize(regex);
  196. var opened = [], isStart = true, i = 0;
  197. function open() {
  198. var t = {value: "(", type: "group.start", isGroup: true};
  199. opened.push(t);
  200. tokens.splice(i++, 0, t);
  201. }
  202. function close() {
  203. var t = {value: ")", type: "group.start"};
  204. t.start = opened.pop();
  205. t.start.end = t;
  206. tokens.splice(i++, 0, t);
  207. }
  208. function tryOpen(){if (isStart) {open(); isStart = false}}
  209. function tryClose(){if (opened.length) close()}
  210. function skip(t) {
  211. var i1 = tokens.indexOf(t.end, i);
  212. if (i1 > i)
  213. i = i1;
  214. }
  215. function lst(t) {return t[t.length - 1]}
  216. function iter(f) {
  217. for (i = 0; i < tokens.length; i++)
  218. f(tokens[i]);
  219. }
  220. function iterGroup(end, f) {
  221. for (var i1 = i + 1; i1 < tokens.length; i1++) {
  222. var t = tokens[i1];
  223. if (t == end)
  224. break;
  225. f(t);
  226. }
  227. }
  228. function peek() { return tokens[i + 1] || {}}
  229. // groupify
  230. iter(function(t){
  231. if (t.type == "group.start") {
  232. tryClose();
  233. isStart = true;
  234. if (!t.hasChildren || t.isSpecial)
  235. skip(t);
  236. } else if (t.type == "group.end") {
  237. isStart = true;
  238. tryClose();
  239. } else if (t.type == "alternation") {
  240. isStart = true;
  241. tryClose();
  242. } else if (t.type != "anchor" && t.type != "quantifier"){
  243. tryOpen();
  244. }
  245. });
  246. tryClose();
  247. // remove redundand groups
  248. var names = [defaultName];
  249. iter(function(t){
  250. if (t.type == "group.start" && !t.isSpecial) {
  251. var captureName = captures[t.number];
  252. if (!t.hasChildren) {
  253. t.tokenName = captureName || lst(names);
  254. skip(t);
  255. } else {
  256. var hasCapture = false;
  257. iterGroup(t.end, function(t1) {
  258. if (t1.type == "group.start" && captures[t1.number])
  259. hasCapture = true;
  260. });
  261. if (hasCapture) {
  262. t.value = "(?:";
  263. if (captureName) {
  264. names.push(captureName);
  265. t.isTokenGroup = true;
  266. }
  267. } else {
  268. t.tokenName = captureName || lst(names);
  269. iterGroup(t.end, function(t1) {
  270. if (t1.value == "(")
  271. t1.value = "(?:";
  272. });
  273. }
  274. }
  275. } else if (t.type == "group.end") {
  276. if (t.start.isTokenGroup)
  277. names.pop();
  278. }
  279. });
  280. // wrap capturing groups with quantifier
  281. iter(function(t){
  282. if (t.type == "group.end" && t.start.value == "(" && peek().type == "quantifier") {
  283. peek().value += ")";
  284. t.start.value += "(?:";
  285. }
  286. });
  287. names = [];
  288. tokens.forEach(function(t) {
  289. if (t.value == "(" || t.value == "((?:" )
  290. t.tokenName && names.push(t.tokenName);
  291. });
  292. return {
  293. names: names,
  294. regex: toStr(tokens)
  295. };
  296. }
  297. /***** converter */
  298. function logDebug(string, obj) {
  299. console.log(string, obj);
  300. }
  301. // tmLanguage processor
  302. // for tracking token states
  303. var states = {start: []};
  304. function processRules(rules){
  305. if (rules.patterns)
  306. states.start = processPatterns(rules.patterns);
  307. if (rules.repository)
  308. processRepository(rules.repository);
  309. return states;
  310. }
  311. function processRepository(r) {
  312. for (var key in r) {
  313. var p = r[key];
  314. if (p.begin)
  315. var stateObj = [processPattern(r[key])];
  316. else if (p.patterns && !p.repository)
  317. var stateObj = processPatterns(p.patterns);
  318. else
  319. var stateObj = [processPattern(r[key])];
  320. if (stateObj)
  321. states["#" + key] = stateObj;
  322. }
  323. }
  324. function processPatterns(pl) {
  325. return pl.map(processPattern);
  326. }
  327. function processPattern(p) {
  328. if (p.end == "(?!\\G)" && p.patterns && p.patterns.length == 1) {
  329. var rule = processPattern(p.patterns[0]);
  330. }
  331. else if (p.begin != null && p.end != null) {
  332. convertBeginEndBackrefs(p);
  333. var rule = simpleRule(p.begin, p.name, p.beginCaptures || p.captures);
  334. var next = processPatterns(p.patterns || []);
  335. var endRule = simpleRule(p.end, p.name, p.endCaptures || p.captures);
  336. endRule.next = "pop";
  337. if (p.applyEndPatternLast)
  338. next.push(endRule);
  339. else
  340. next.unshift(endRule);
  341. if (p.name || p.contentName)
  342. next.push({defaultToken: p.name || p.contentName});
  343. rule.push = next;
  344. rule = removeIncludeSelf(rule);
  345. }
  346. else if (p.match) {
  347. var rule = simpleRule(p.match, p.name, p.captures);
  348. }
  349. else if (p.include) {
  350. var rule = {include: p.include};
  351. }
  352. else {
  353. var rule = {todo: p};
  354. }
  355. if (p.comment)
  356. rule.comment = (rule.comment || "") + p.comment;
  357. if (p.repository)
  358. processRepository(p.repository);
  359. return rule;
  360. }
  361. function simpleRule(regex, name, captures) {
  362. name = name || "text";
  363. var rule = {token: "", regex: ""};
  364. var origRegex = regex;
  365. regex = transformRegExp(origRegex, rule);
  366. if (captures) {
  367. var tokenArray = [];
  368. Object.keys(captures).forEach(function(x){
  369. tokenArray[x] = captures[x] && captures[x].name;
  370. });
  371. if (tokenArray.length == 1) {
  372. name = tokenArray[0];
  373. } else {
  374. var fixed = fixGroups(tokenArray, name, regex);
  375. name = fixed.names;
  376. regex = fixed.regex;
  377. if (name.length == 1)
  378. name = name[0];
  379. }
  380. }
  381. if (typeof name == "string")
  382. regex = convertToNonCapturingGroups(regex);
  383. regex = simplifyNonCapturingGroups(regex);
  384. try {new RegExp(regex);} catch(e) {
  385. rule.TODO = "FIXME: regexp doesn't have js equivalent";
  386. rule.originalRegex = origRegex;
  387. // lookbehinds are mostly used to force ordering
  388. // regex = removeLookBehinds(regex);
  389. }
  390. rule.token = name;
  391. rule.regex = regex;
  392. return rule;
  393. }
  394. function removeIncludeSelf(rule) {
  395. if (!rule.push)
  396. return rule;
  397. var hasSelfInclude = false;
  398. var escapeRule = null;
  399. var complexSelfInclude = false;
  400. rule.push.forEach(function(sub) {
  401. if (sub.include == "$self") {
  402. hasSelfInclude = true;
  403. } else if (sub.defaultToken) {
  404. return;
  405. } else if (sub.next == "pop") {
  406. escapeRule = sub;
  407. } else
  408. complexSelfInclude = true;
  409. });
  410. if (hasSelfInclude) {
  411. console.warn("can't convert include $self");
  412. return {todo: rule};
  413. if (complexSelfInclude) {
  414. console.warn("can't convert include $self");
  415. rule.toDo = "include $self not fully supported";
  416. return rule;
  417. }
  418. console.warn("include $self not fully supported");
  419. delete rule.push;
  420. delete escapeRule.next;
  421. rule.includeSelf = true;
  422. escapeRule.includeSelf = true;
  423. return [rule, escapeRule];
  424. }
  425. return rule;
  426. }
  427. // regex transformation
  428. function removeXFlag(str) {
  429. var tokens = tokenize(str);
  430. return toStr(tokens);
  431. }
  432. function transformRegExp(str, rule) {
  433. str = convertNewLinesTo$(str);
  434. str = removeInlineFlags(str, rule);
  435. str = str.replace(/(\\[xu]){([a-fA-F\d]+)}/g, '$1$2');
  436. str = convertCharacterTypes(str, rule);
  437. checkForNamedCaptures(str);
  438. return str;
  439. }
  440. //
  441. function extractPatterns(tmRules) {
  442. return processRules(tmRules);
  443. }
  444. function detectLoops(states) {
  445. var data = {};
  446. var keys = Object.keys(states);
  447. var flattenedStates = {};
  448. function addRef(item, name) {
  449. if (item.refs.indexOf(name) == -1)
  450. item.refs.push(name);
  451. }
  452. function anonStateId(name, next) {
  453. var i = 0, old = name;
  454. while (flattenedStates[name] || states[name]) {
  455. name = old + "_" + i++;
  456. }
  457. // console.log(old, name)
  458. return name;
  459. }
  460. function addState(key, rules) {
  461. if (rules && !flattenedStates[key])
  462. flattenedStates[key] = rules;
  463. return rules || flattenedStates[key];
  464. }
  465. for (var i = 0; i < keys.length; i++) {
  466. var key = keys[i];
  467. var state = addState(key, states[key]);
  468. var item = data[key] || (data[key] = {/* name: key, */ refs: []});
  469. state.forEach(function(rule) {
  470. var next = rule.push || rule.next;
  471. if (next == "pop") {
  472. // nothing
  473. } else if (typeof next == "string") {
  474. addRef(item, next);
  475. } else if (next) {
  476. var anonId = anonStateId(key, next);
  477. addState(anonId, next);
  478. if (rule.push)
  479. addRef(item, anonId);
  480. keys.push(anonId);
  481. } else if (rule.include) {
  482. addRef(item, rule.include);
  483. }
  484. });
  485. }
  486. var cycles = [];
  487. function addPath(start, path) {
  488. var node = data[start];
  489. path.push(start);
  490. if (!node || !node.refs)
  491. console.log(start);
  492. var i = path.indexOf(start);
  493. if (i > -1 && i != path.length - 1 || start == "$self" || start == "$base") {
  494. if (i != -1)
  495. path = path.slice(i);
  496. for (var j = 0; j < cycles.length; j++) {
  497. if (cycles[j] + "" == path + "")
  498. return;
  499. }
  500. return cycles.push(path);
  501. }
  502. if (!node || !node.refs || !node.refs.length || path.length>30)
  503. return;
  504. node.refs.forEach(function(x) {
  505. addPath(x, path.concat());
  506. });
  507. }
  508. addPath("start", []);
  509. console.error(cycles.join("\n"));
  510. }
  511. function test(fileName) {
  512. console.log("testing highlighter");
  513. try {
  514. var module = require(fileName);
  515. var Mode = module[Object.keys(module)[0]];
  516. var mode = new Mode();
  517. mode.getTokenizer().getLineTokens("hello world");
  518. } catch(e) {
  519. console.log(e);
  520. }
  521. }
  522. function guessComment(patterns) {
  523. var comment = {};
  524. for (var i in patterns) {
  525. var state = patterns[i];
  526. state.forEach(function(r) {
  527. if (typeof r.token == "string") {
  528. if (/\bcomment\b/.test(r.token)) {
  529. comment.line = r.regex;
  530. }
  531. }
  532. });
  533. }
  534. return comment;
  535. }
  536. // cli stuff
  537. var modeTemplate = fs.readFileSync(__dirname + "/templates/mode.js", "utf8");
  538. var modeHighlightTemplate = fs.readFileSync(__dirname + "/templates/highlight_rules.js", "utf8");
  539. function fetchAndConvert(name) {
  540. console.log("Converting " + name);
  541. if (/^http/.test(name)) {
  542. if (/:\/\/github.com/.test(name)) {
  543. name = name.replace(/\/blob\//, "/").replace("github.com", "raw.github.com");
  544. }
  545. return lib.download(name, function(data) {
  546. convertTmLanguage(name, data);
  547. });
  548. }
  549. var path = /^(\/|\w:)/.test(name) ? name : process.cwd() + "/" + name;
  550. var langStr = fs.readFileSync(path, "utf8");
  551. convertTmLanguage(name, langStr);
  552. }
  553. function convertTmLanguage(name, langStr) {
  554. parseLanguage(langStr, function(language) {
  555. var highlighterFilename = lib.snakeCase(language.name).replace(/[^\w]/g, "");
  556. var languageNameSanitized = lib.camelCase(language.name).replace(/[^\w]/g, "");
  557. require("./add_mode")(languageNameSanitized, (language.fileTypes || []).join("|"));
  558. var highlighterFile = pathlib.normalize(lib.AceLib + "ace/mode/" + highlighterFilename + "_highlight_rules.js");
  559. var modeFile = pathlib.normalize(lib.AceLib + "ace/mode/" + highlighterFilename + ".js");
  560. if (devMode) {
  561. console.log(util.inspect(language.patterns, false, 4));
  562. console.log(util.inspect(language.repository, false, 4));
  563. }
  564. var patterns = extractPatterns(language);
  565. detectLoops(patterns);
  566. // var uuid = language.uuid
  567. delete language.uuid;
  568. delete language.patterns;
  569. delete language.repository;
  570. var comment = guessComment(patterns);
  571. var languageMode = lib.fillTemplate(modeTemplate, {
  572. language: languageNameSanitized,
  573. languageHighlightFilename: highlighterFilename,
  574. lineCommentStart: JSON.stringify(comment.line || "//"),
  575. blockCommentStart: JSON.stringify(comment.start || "/*"),
  576. blockCommentEnd: JSON.stringify(comment.end || "*/")
  577. });
  578. var languageHighlightRules = lib.fillTemplate(modeHighlightTemplate, {
  579. language: languageNameSanitized,
  580. languageTokens: lib.formatJS(patterns, " ").trim(),
  581. uuid: language.uuid,
  582. name: name,
  583. metaData: lib.formatJS(language, "").trim()
  584. });
  585. if (devMode) {
  586. console.log(languageMode);
  587. console.log(languageHighlightRules);
  588. console.log("Not writing, 'cause we're in dev mode, baby.");
  589. }
  590. else {
  591. fs.writeFileSync(highlighterFile, languageHighlightRules);
  592. fs.writeFileSync(modeFile, languageMode);
  593. console.log("created file " + highlighterFile);
  594. test(modeFile);
  595. }
  596. });
  597. }
  598. if (!module.parent) {
  599. var args = process.argv.splice(2);
  600. var devMode = args[0] == "--dev";
  601. if (devMode)
  602. args.shift();
  603. if (args.length < 1) {
  604. console.error("Usage: node tmlanguage.js [--dev] path/or/url/to/syntax.file ...");
  605. process.exit(1);
  606. }
  607. args.forEach(fetchAndConvert);
  608. } else {
  609. exports.fetchAndConvert = fetchAndConvert;
  610. }