| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697 |
- require("amd-loader");
- var fs = require("fs");
- var util = require("util");
- var lib = require("./lib");
- var pathlib = require("path");
- var parseLanguage = lib.parsePlist;
- var tk = require("./regexp_tokenizer");
- var tokenize = tk.tokenize;
- var toStr = tk.toStr;
- function last(array) {return array[array.length - 1]}
- function convertHexEscape(tokens) {
- var inChClass = false;
- tokens.forEach(function(t) {
- if (t.type == "charclass")
- inChClass = true;
- else if (t.type == "charclass.end")
- inChClass = false;
- else if (t.type == "charType"){
- if (t.value == "\\h") {
- t.type = "text";
- t.value = inChClass ? "\\da-fA-F" : "[\\da-fA-F]";
- }
- else if (t.value == "\\H") {
- if (inChClass) {
- console.warn("can't convert \\H in charclass");
- return;
- }
- t.type = "text";
- t.value = "[^\\da-fA-F]";
- }
- }
- });
- return tokens;
- }
- function convertNewLinesTo$(str) {
- var tokens = tokenize(str);
- for (var i = 0; i < tokens.length; i++) {
- var t= tokens[i];
- if (t.type == "char" && t.value == "\\n") {
- var p = tokens[i + 1] || {};
- if (p.type != "quantifier") {
- t.value = "$";
- while (p.value == "\\n" || p.type == "quantifier") {
- p.value = "";
- p = tokens[++i + 1] || {};
- }
- } else if (/\?|\*|{,|{0,/.test(p.value)) {
- t.value = p.value = "";
- } else
- p.value = "";
- }
- }
- return toStr(tokens).replace(/[$]+/g, "$");
- }
- function convertCharacterTypes(str) {
- var tokens = tokenize(str);
- tokens = convertHexEscape(tokens);
- var warn = false;
- tokens.forEach(function(t){
- if (t.type == "quantifier") {
- var val = t.value;
- if (val.slice(-1) == "+" && val.length > 1) {
- t.value = val.slice(0, -1);
- warn = val;
- }
- }
- });
- if (warn)
- console.log("converted possesive quantifier " + warn + " to *");
- return toStr(tokens);
- }
- function removeInlineFlags(str, rule) {
- var tokens = tokenize(str);
- var caseInsensitive = false;
- tokens.forEach(function(t, i) {
- if (t.type == "group.start" && /[imsx]/.test(t.value)) {
- if (/i/.test(t.value))
- caseInsensitive = true;
- t.value = t.value.replace(/[imsx\-]/g, "");
- var next = tokens[i + 1];
- if (next && next.type == "group.end") {
- t.value = next.value = "";
- }
- }
- });
- if (caseInsensitive && rule)
- rule.caseInsensitive = true;
- return toStr(tokens);
- }
- function convertToNonCapturingGroups(str) {
- var tokens = tokenize(str);
- tokens.forEach(function(t, i) {
- if (t.type == "group.start" && t.value == "(")
- t.value += "?:";
- });
- return toStr(tokens);
- }
- function simplifyNonCapturingGroups(str) {
- var tokens = tokenize(str);
- var t = tokens[0] || {};
- if (t.type == "group.start" && t.value == "(?:"
- && t.end == last(tokens)) {
- t.value = t.end.value = "";
- }
- var i = 0;
- function iter(f) {
- for (i = 0; i < tokens.length; i++)
- f(tokens[i]);
- }
- function iterGroup(end, f) {
- for (var i1 = i + 1; i1 < tokens.length; i1++) {
- var t = tokens[i1];
- if (t == end)
- break;
- var index = f && f(t);
- if (index > i1)
- i1 = index;
- }
- return i1;
- }
- iter(function (t) {
- if (t.type == "group.start" && t.value == "(?:") {
- if (!t.end)
- return console.error("malformed regex: " + str);
- var canRemove = true;
- var next = tokens[tokens.indexOf(t.end, i) + 1];
- if (next && next.type == "quantifier")
- return;
- iterGroup(t.end, function(t) {
- if (t.type == "alternation")
- canRemove = false;
- else if (t.type == "group.start" && t.end)
- return iterGroup(t.end);
- });
- if (canRemove)
- t.value = t.end.value = "";
- }
- });
- return toStr(tokens);
- }
- function removeLookBehinds(str) {
- var tokens = tokenize(str);
- var toRemove = null;
- tokens.forEach(function(t, i) {
- if (!toRemove && t.type == "group.start" && /</.test(t.value)) {
- toRemove = t.end;
- toRemove.content = [];
- }
- if (toRemove) {
- toRemove.content.push(t.value);
- t.value = "";
- }
- if (t == toRemove) {
- var c = toRemove.content.slice(1, -1).join("");
- if (/\^/.test(c))
- toRemove.value = "(?:" + c +")";
- toRemove = null;
- }
- });
- return toStr(tokens);
- }
- function convertBeginEndBackrefs(rule) {
- if (!/\\\d/.test(rule.end))
- return;
- var startTokens = tokenize(rule.begin);
- var endTokens = tokenize(rule.end);
- var groups = {};
- startTokens.forEach(function(t, i) {
- if (t.number && t.end && t.type == "group.start") {
- var endIndex = startTokens.indexOf(t.end, i + 1);
- var content = startTokens.slice(i+1, endIndex);
- groups[t.number] = toStr(content);
- }
- });
- endTokens.forEach(function(t) {
- if (t.type == "backRef") {
- var num = t.value.substr(1);
- if (groups[num])
- t.value = "(?:" + groups[num] + ")";
- }
- });
- rule.end = toStr(endTokens);
- console.warn("Begin-End-Backreference is detected", rule);
- }
- function checkForNamedCaptures(str) {
- var tokens = tokenize(str);
- tokens.forEach(function(t) {
- if (t.type == "group.start" && t.name)
- console.warn("named capture not implemented", str);
- if (t.type == "backRef")
- console.warn("backRef not implemented ", str);
- });
- }
- function fixGroups(captures, defaultName, regex) {
- var tokens = tokenize(regex);
- var opened = [], isStart = true, i = 0;
- function open() {
- var t = {value: "(", type: "group.start", isGroup: true};
- opened.push(t);
- tokens.splice(i++, 0, t);
- }
- function close() {
- var t = {value: ")", type: "group.start"};
- t.start = opened.pop();
- t.start.end = t;
- tokens.splice(i++, 0, t);
- }
- function tryOpen(){if (isStart) {open(); isStart = false}}
- function tryClose(){if (opened.length) close()}
- function skip(t) {
- var i1 = tokens.indexOf(t.end, i);
- if (i1 > i)
- i = i1;
- }
- function lst(t) {return t[t.length - 1]}
- function iter(f) {
- for (i = 0; i < tokens.length; i++)
- f(tokens[i]);
- }
- function iterGroup(end, f) {
- for (var i1 = i + 1; i1 < tokens.length; i1++) {
- var t = tokens[i1];
- if (t == end)
- break;
- f(t);
- }
- }
- function peek() { return tokens[i + 1] || {}}
- // groupify
- iter(function(t){
- if (t.type == "group.start") {
- tryClose();
- isStart = true;
- if (!t.hasChildren || t.isSpecial)
- skip(t);
- } else if (t.type == "group.end") {
- isStart = true;
- tryClose();
- } else if (t.type == "alternation") {
- isStart = true;
- tryClose();
- } else if (t.type != "anchor" && t.type != "quantifier"){
- tryOpen();
- }
- });
- tryClose();
- // remove redundand groups
- var names = [defaultName];
- iter(function(t){
- if (t.type == "group.start" && !t.isSpecial) {
- var captureName = captures[t.number];
- if (!t.hasChildren) {
- t.tokenName = captureName || lst(names);
- skip(t);
- } else {
- var hasCapture = false;
- iterGroup(t.end, function(t1) {
- if (t1.type == "group.start" && captures[t1.number])
- hasCapture = true;
- });
- if (hasCapture) {
- t.value = "(?:";
- if (captureName) {
- names.push(captureName);
- t.isTokenGroup = true;
- }
- } else {
- t.tokenName = captureName || lst(names);
- iterGroup(t.end, function(t1) {
- if (t1.value == "(")
- t1.value = "(?:";
- });
- }
- }
- } else if (t.type == "group.end") {
- if (t.start.isTokenGroup)
- names.pop();
- }
- });
- // wrap capturing groups with quantifier
- iter(function(t){
- if (t.type == "group.end" && t.start.value == "(" && peek().type == "quantifier") {
- peek().value += ")";
- t.start.value += "(?:";
- }
- });
- names = [];
- tokens.forEach(function(t) {
- if (t.value == "(" || t.value == "((?:" )
- t.tokenName && names.push(t.tokenName);
- });
- return {
- names: names,
- regex: toStr(tokens)
- };
- }
- /***** converter */
- function logDebug(string, obj) {
- console.log(string, obj);
- }
- // tmLanguage processor
- // for tracking token states
- var states = {start: []};
- function processRules(rules){
- if (rules.patterns)
- states.start = processPatterns(rules.patterns);
- if (rules.repository)
- processRepository(rules.repository);
- return states;
- }
- function processRepository(r) {
- for (var key in r) {
- var p = r[key];
- if (p.begin)
- var stateObj = [processPattern(r[key])];
- else if (p.patterns && !p.repository)
- var stateObj = processPatterns(p.patterns);
- else
- var stateObj = [processPattern(r[key])];
- if (stateObj)
- states["#" + key] = stateObj;
- }
- }
- function processPatterns(pl) {
- return pl.map(processPattern);
- }
- function processPattern(p) {
- if (p.end == "(?!\\G)" && p.patterns && p.patterns.length == 1) {
- var rule = processPattern(p.patterns[0]);
- }
- else if (p.begin != null && p.end != null) {
- convertBeginEndBackrefs(p);
- var rule = simpleRule(p.begin, p.name, p.beginCaptures || p.captures);
- var next = processPatterns(p.patterns || []);
- var endRule = simpleRule(p.end, p.name, p.endCaptures || p.captures);
- endRule.next = "pop";
- if (p.applyEndPatternLast)
- next.push(endRule);
- else
- next.unshift(endRule);
- if (p.name || p.contentName)
- next.push({defaultToken: p.name || p.contentName});
- rule.push = next;
- rule = removeIncludeSelf(rule);
- }
- else if (p.match) {
- var rule = simpleRule(p.match, p.name, p.captures);
- }
- else if (p.include) {
- var rule = {include: p.include};
- }
- else {
- var rule = {todo: p};
- }
- if (p.comment)
- rule.comment = (rule.comment || "") + p.comment;
- if (p.repository)
- processRepository(p.repository);
- return rule;
- }
- function simpleRule(regex, name, captures) {
- name = name || "text";
- var rule = {token: "", regex: ""};
- var origRegex = regex;
- regex = transformRegExp(origRegex, rule);
- if (captures) {
- var tokenArray = [];
- Object.keys(captures).forEach(function(x){
- tokenArray[x] = captures[x] && captures[x].name;
- });
- if (tokenArray.length == 1) {
- name = tokenArray[0];
- } else {
- var fixed = fixGroups(tokenArray, name, regex);
- name = fixed.names;
- regex = fixed.regex;
- if (name.length == 1)
- name = name[0];
- }
- }
- if (typeof name == "string")
- regex = convertToNonCapturingGroups(regex);
- regex = simplifyNonCapturingGroups(regex);
- try {new RegExp(regex);} catch(e) {
- rule.TODO = "FIXME: regexp doesn't have js equivalent";
- rule.originalRegex = origRegex;
- // lookbehinds are mostly used to force ordering
- // regex = removeLookBehinds(regex);
- }
- rule.token = name;
- rule.regex = regex;
- return rule;
- }
- function removeIncludeSelf(rule) {
- if (!rule.push)
- return rule;
- var hasSelfInclude = false;
- var escapeRule = null;
- var complexSelfInclude = false;
- rule.push.forEach(function(sub) {
- if (sub.include == "$self") {
- hasSelfInclude = true;
- } else if (sub.defaultToken) {
- return;
- } else if (sub.next == "pop") {
- escapeRule = sub;
- } else
- complexSelfInclude = true;
- });
- if (hasSelfInclude) {
- console.warn("can't convert include $self");
- return {todo: rule};
-
- if (complexSelfInclude) {
- console.warn("can't convert include $self");
- rule.toDo = "include $self not fully supported";
- return rule;
- }
- console.warn("include $self not fully supported");
- delete rule.push;
- delete escapeRule.next;
- rule.includeSelf = true;
- escapeRule.includeSelf = true;
- return [rule, escapeRule];
- }
- return rule;
- }
- // regex transformation
- function removeXFlag(str) {
- var tokens = tokenize(str);
- return toStr(tokens);
- }
- function transformRegExp(str, rule) {
- str = convertNewLinesTo$(str);
- str = removeInlineFlags(str, rule);
- str = str.replace(/(\\[xu]){([a-fA-F\d]+)}/g, '$1$2');
- str = convertCharacterTypes(str, rule);
- checkForNamedCaptures(str);
- return str;
- }
- //
- function extractPatterns(tmRules) {
- return processRules(tmRules);
- }
- function detectLoops(states) {
- var data = {};
- var keys = Object.keys(states);
- var flattenedStates = {};
- function addRef(item, name) {
- if (item.refs.indexOf(name) == -1)
- item.refs.push(name);
- }
- function anonStateId(name, next) {
- var i = 0, old = name;
- while (flattenedStates[name] || states[name]) {
- name = old + "_" + i++;
- }
- // console.log(old, name)
- return name;
- }
- function addState(key, rules) {
- if (rules && !flattenedStates[key])
- flattenedStates[key] = rules;
- return rules || flattenedStates[key];
- }
-
-
- for (var i = 0; i < keys.length; i++) {
- var key = keys[i];
- var state = addState(key, states[key]);
-
- var item = data[key] || (data[key] = {/* name: key, */ refs: []});
- state.forEach(function(rule) {
- var next = rule.push || rule.next;
- if (next == "pop") {
- // nothing
- } else if (typeof next == "string") {
- addRef(item, next);
- } else if (next) {
- var anonId = anonStateId(key, next);
- addState(anonId, next);
- if (rule.push)
- addRef(item, anonId);
- keys.push(anonId);
- } else if (rule.include) {
- addRef(item, rule.include);
- }
- });
- }
-
-
- var cycles = [];
- function addPath(start, path) {
- var node = data[start];
- path.push(start);
- if (!node || !node.refs)
- console.log(start);
- var i = path.indexOf(start);
- if (i > -1 && i != path.length - 1 || start == "$self" || start == "$base") {
- if (i != -1)
- path = path.slice(i);
- for (var j = 0; j < cycles.length; j++) {
- if (cycles[j] + "" == path + "")
- return;
- }
- return cycles.push(path);
- }
-
- if (!node || !node.refs || !node.refs.length || path.length>30)
- return;
- node.refs.forEach(function(x) {
- addPath(x, path.concat());
- });
- }
- addPath("start", []);
-
- console.error(cycles.join("\n"));
- }
- function test(fileName) {
- console.log("testing highlighter");
- try {
- var module = require(fileName);
- var Mode = module[Object.keys(module)[0]];
- var mode = new Mode();
- mode.getTokenizer().getLineTokens("hello world");
- } catch(e) {
- console.log(e);
- }
- }
- function guessComment(patterns) {
- var comment = {};
- for (var i in patterns) {
- var state = patterns[i];
- state.forEach(function(r) {
- if (typeof r.token == "string") {
- if (/\bcomment\b/.test(r.token)) {
- comment.line = r.regex;
- }
- }
- });
- }
- return comment;
- }
- // cli stuff
- var modeTemplate = fs.readFileSync(__dirname + "/templates/mode.js", "utf8");
- var modeHighlightTemplate = fs.readFileSync(__dirname + "/templates/highlight_rules.js", "utf8");
- function fetchAndConvert(name) {
- console.log("Converting " + name);
- if (/^http/.test(name)) {
- if (/:\/\/github.com/.test(name)) {
- name = name.replace(/\/blob\//, "/").replace("github.com", "raw.github.com");
- }
- return lib.download(name, function(data) {
- convertTmLanguage(name, data);
- });
- }
- var path = /^(\/|\w:)/.test(name) ? name : process.cwd() + "/" + name;
- var langStr = fs.readFileSync(path, "utf8");
- convertTmLanguage(name, langStr);
- }
- function convertTmLanguage(name, langStr) {
- parseLanguage(langStr, function(language) {
- var highlighterFilename = lib.snakeCase(language.name).replace(/[^\w]/g, "");
- var languageNameSanitized = lib.camelCase(language.name).replace(/[^\w]/g, "");
-
- require("./add_mode")(languageNameSanitized, (language.fileTypes || []).join("|"));
- var highlighterFile = pathlib.normalize(lib.AceLib + "ace/mode/" + highlighterFilename + "_highlight_rules.js");
- var modeFile = pathlib.normalize(lib.AceLib + "ace/mode/" + highlighterFilename + ".js");
- if (devMode) {
- console.log(util.inspect(language.patterns, false, 4));
- console.log(util.inspect(language.repository, false, 4));
- }
- var patterns = extractPatterns(language);
- detectLoops(patterns);
-
- // var uuid = language.uuid
- delete language.uuid;
- delete language.patterns;
- delete language.repository;
- var comment = guessComment(patterns);
- var languageMode = lib.fillTemplate(modeTemplate, {
- language: languageNameSanitized,
- languageHighlightFilename: highlighterFilename,
- lineCommentStart: JSON.stringify(comment.line || "//"),
- blockCommentStart: JSON.stringify(comment.start || "/*"),
- blockCommentEnd: JSON.stringify(comment.end || "*/")
- });
- var languageHighlightRules = lib.fillTemplate(modeHighlightTemplate, {
- language: languageNameSanitized,
- languageTokens: lib.formatJS(patterns, " ").trim(),
- uuid: language.uuid,
- name: name,
- metaData: lib.formatJS(language, "").trim()
- });
- if (devMode) {
- console.log(languageMode);
- console.log(languageHighlightRules);
- console.log("Not writing, 'cause we're in dev mode, baby.");
- }
- else {
- fs.writeFileSync(highlighterFile, languageHighlightRules);
- fs.writeFileSync(modeFile, languageMode);
- console.log("created file " + highlighterFile);
- test(modeFile);
- }
- });
- }
- if (!module.parent) {
- var args = process.argv.splice(2);
- var devMode = args[0] == "--dev";
- if (devMode)
- args.shift();
- if (args.length < 1) {
- console.error("Usage: node tmlanguage.js [--dev] path/or/url/to/syntax.file ...");
- process.exit(1);
- }
- args.forEach(fetchAndConvert);
- } else {
- exports.fetchAndConvert = fetchAndConvert;
- }
|