Преглед изворни кода

[spark] Java and Scala support

Still missing: syntax highlighting is Python and hints are missing
Romain Rigaux пре 12 година
родитељ
комит
34a106ff7c

+ 11 - 8
apps/spark/src/spark/api.py

@@ -24,7 +24,7 @@ from django.core.urlresolvers import reverse
 from django.utils.html import escape
 from django.utils.translation import ugettext as _
 
-from spark.conf import SPARK_HOME
+from spark.conf import SPARK_HOME, SPARK_MASTER
 from desktop.lib.view_util import format_duration_in_millis
 from filebrowser.views import location_to_url
 from jobbrowser.views import job_single_logs
@@ -56,7 +56,7 @@ class OozieSparkApi:
 
   def submit(self, spark_script, params):
     mapping = {
-      'oozie.use.system.libpath':  'false',
+      'oozie.use.system.libpath': 'false',
     }
 
     workflow = None
@@ -77,7 +77,7 @@ class OozieSparkApi:
     workflow.save()
     Workflow.objects.initialize(workflow, self.fs)
 
-    spark_script_path = workflow.deployment_dir + '/spark.py'
+    spark_script_path = workflow.deployment_dir + '/spark.script'
     spark_launcher_path = workflow.deployment_dir + '/spark.sh'
     self.fs.do_as_user(self.user.username, self.fs.create, spark_script_path, data=spark_script.dict['script'])
     self.fs.do_as_user(self.user.username, self.fs.create, spark_launcher_path, data="""
@@ -85,11 +85,14 @@ class OozieSparkApi:
 
 WORKSPACE=`pwd`
 cd %(spark_home)s
-MASTER=spark://runreal:7077 pyspark $WORKSPACE/spark.py
-    """ % {'spark_home': SPARK_HOME.get()})
-# MASTER=spark://kostas-1.ent.cloudera.com:7077 pyspark $WORKSPACE/spark.py
-
-    files = ['spark.py', 'spark.sh']
+%(spark_master)s %(spark_shell)s $WORKSPACE/spark.script
+    """ % {
+        'spark_home': SPARK_HOME.get(),
+        'spark_master': 'MASTER=%s' % SPARK_MASTER.get() if SPARK_MASTER.get() != '' else '',
+        'spark_shell': 'pyspark' if spark_script.dict['language'] == 'python' else 'spark-shell <'
+    })
+
+    files = ['spark.script', 'spark.sh']
     archives = []
 
     popup_params = json.loads(params)

+ 7 - 0
apps/spark/src/spark/conf.py

@@ -21,6 +21,13 @@ from django.utils.translation import ugettext as _, ugettext_lazy as _t
 from desktop.lib.conf import Config, validate_path
 
 
+SPARK_MASTER = Config(
+  key="spark_master",
+  help=_t("Address of the Spark master, e.g spark://localhost:7077. If empty use the current configuration. "
+          "Can be overriden in the script too."),
+  default=""
+)
+
 SPARK_HOME = Config(
   key="spark_home",
   help=_t("Local path to Spark Home on all the nodes of the cluster."),

+ 6 - 4
apps/spark/src/spark/models.py

@@ -31,7 +31,7 @@ from hadoop.fs.hadoopfs import Hdfs
 
 
 class SparkScript(models.Model):
-  _ATTRIBUTES = ['script', 'name', 'properties', 'job_id', 'parameters', 'resources', 'hadoopProperties', 'type']
+  _ATTRIBUTES = ['script', 'name', 'properties', 'job_id', 'parameters', 'resources', 'hadoopProperties', 'language']
 
   data = models.TextField(default=json.dumps({
       'script': '',
@@ -41,7 +41,7 @@ class SparkScript(models.Model):
       'parameters': [],
       'resources': [],
       'hadoopProperties': [],
-      'type': 'python',
+      'language': 'python',
   }))
 
   doc = generic.GenericRelation(Doc, related_name='spark_doc')
@@ -66,7 +66,7 @@ class SparkScript(models.Model):
     return reverse('spark:index') + '#edit/%s' % self.id
 
 
-def create_or_update_script(id, name, script, user, parameters, resources, hadoopProperties, is_design=True):
+def create_or_update_script(id, name, script, user, parameters, resources, hadoopProperties, language, is_design=True):
   try:
     spark_script = SparkScript.objects.get(id=id)
     spark_script.doc.get().can_read_or_exception(user)
@@ -81,7 +81,8 @@ def create_or_update_script(id, name, script, user, parameters, resources, hadoo
       'script': script,
       'parameters': parameters,
       'resources': resources,
-      'hadoopProperties': hadoopProperties
+      'hadoopProperties': hadoopProperties,
+      'language': language,
   })
 
   return spark_script
@@ -103,6 +104,7 @@ def get_scripts(user, is_design=None):
       'parameters': data['parameters'],
       'resources': data['resources'],
       'hadoopProperties': data.get('hadoopProperties', []),
+      'language': data['language'],
       'isDesign': not script.doc.get().is_historic(),
     }
     scripts.append(massaged_script)

+ 41 - 9
apps/spark/src/spark/templates/app.mako

@@ -113,7 +113,7 @@ ${ commonheader(None, "spark", user) | n,unicode }
           <ul class="nav nav-list">
             <li class="nav-header">${_('Editor')}</li>
             <li data-bind="click: editScript" class="active" data-section="edit">
-              <a href="#"><i class="fa fa-edit"></i><span data-bind="value: currentScript().type"> ${ _('Python') }</span></a>
+              <a href="#"><i class="fa fa-edit"></i> <span data-bind="text: viewModel.currentScript().language" style="text-transform:capitalize"></span></a>
             </li>
             <li data-bind="click: editScriptProperties" data-section="properties">
               <a href="#"><i class="fa fa-reorder"></i> ${ _('Properties') }</a>
@@ -140,6 +140,11 @@ ${ commonheader(None, "spark", user) | n,unicode }
               </a>
             </li>
             <li class="nav-header">${_('File')}</li>
+            <li data-bind="click: confirmNewScript">
+              <a href="#" title="${ _('New script') }" rel="tooltip" data-placement="right">
+                <i class="fa fa-plus-circle"></i> ${ _('New') }
+              </a>
+            </li>
             <li data-bind="visible: currentScript().id() != -1, click: copyScript">
               <a href="#" title="${ _('Copy the script') }" rel="tooltip" data-placement="right">
                 <i class="fa fa-files-o"></i> ${ _('Copy') }
@@ -150,11 +155,6 @@ ${ commonheader(None, "spark", user) | n,unicode }
                 <i class="fa fa-trash-o"></i> ${ _('Delete') }
               </a>
             </li>
-            <li data-bind="click: confirmNewScript">
-              <a href="#" title="${ _('New script') }" rel="tooltip" data-placement="right">
-                <i class="fa fa-plus-circle"></i> ${ _('Script') }
-              </a>
-            </li>
             <li>
             <a href="#" id="help">
               <i class="fa fa-question-circle"></i>
@@ -476,6 +476,33 @@ ${ commonheader(None, "spark", user) | n,unicode }
   </div>
 </div>
 
+<div id="newScriptModal" class="modal hide fade">
+  <div class="modal-header">
+    <a href="#" class="close" data-dismiss="modal">&times;</a>
+    <h3>${_('New script')}</h3>
+  </div>
+  <div class="modal-body">
+    <p>
+      ${_('Please select the language of your script:')}<br/><br/>
+      <label>
+        ${ _('Java') } &nbsp;
+        <input type="radio" checked name="languages" value="java" data-bind="checked: currentScript().language" />
+      </label>
+      <label>
+        ${ _('Scala') } &nbsp;
+        <input type="radio" name="languages" value="scala" data-bind="checked: currentScript().language" />
+      </label>
+      <label>
+        ${ _('Python') } &nbsp;
+        <input type="radio" name="languages" value="python" data-bind="checked: currentScript().language" />
+      </label>
+    </p>
+  </div>
+  <div class="modal-footer">
+    <a class="btn" data-dismiss="modal">${_('Cancel')}</a>
+    <button class="btn btn-primary disable-feedback" data-bind="click: createAndEditScript">${_('Create')}</button>
+  </div>
+</div>
 
 <div class="bottomAlert alert"></div>
 
@@ -484,6 +511,7 @@ ${ commonheader(None, "spark", user) | n,unicode }
 <script src="/spark/static/js/spark.ko.js" type="text/javascript" charset="utf-8"></script>
 <script src="/static/ext/js/routie-0.3.0.min.js" type="text/javascript" charset="utf-8"></script>
 <script src="/static/ext/js/codemirror-3.11.js"></script>
+<script src="/static/js/clike.js"></script>
 <script src="/static/js/codemirror-python.js"></script>
 <script src="/static/js/codemirror-show-hint.js"></script>
 <script src="/static/js/codemirror-pig-hint.js"></script>
@@ -505,7 +533,11 @@ ${ commonheader(None, "spark", user) | n,unicode }
     TOOLTIP_STOP: "${ _('Stop execution.') }",
     SAVED: "${ _('Saved') }",
     NEW_SCRIPT_NAME: "${ _('Unsaved script') }",
-    NEW_SCRIPT_CONTENT: "Example:\nfrom pyspark import SparkContext\n\nsc = SparkContext('local', 'App Name')\n\n\nwords = sc.textFile('/usr/share/dict/words')\nprint words.filter(lambda w: w.startswith('spar')).take(5)",
+    NEW_SCRIPT_CONTENT: {
+      'python': "Example:\nfrom pyspark import SparkContext\n\nsc = SparkContext('local', 'App Name')\n\n\nwords = sc.textFile('/usr/share/dict/words')\nprint words.filter(lambda w: w.startswith('spar')).take(5)",
+      'java': 'Example:\npublic class JavaWordCount {\n  ...\n}',
+      'scala': 'Example:\nval distFile = sc.textFile("data.txt")\nval linesWithSpark = distFile.filter(line => line.contains("Spark"))\nprintln(linesWithSpark)'
+    },
     NEW_SCRIPT_PARAMETERS: [],
     NEW_SCRIPT_RESOURCES: [],
     NEW_SCRIPT_HADOOP_PROPERTIES: []
@@ -628,7 +660,7 @@ ${ commonheader(None, "spark", user) | n,unicode }
       value: scriptEditor.value,
       readOnly: false,
       lineNumbers: true,
-      mode: "text/x-python",
+      mode: "text/x-" + viewModel.currentScript().language(), // TODO update to language dynamically
       extraKeys: {
         "Ctrl-Space": "autocomplete",
         "Ctrl-Enter": function () {
@@ -706,7 +738,7 @@ ${ commonheader(None, "spark", user) | n,unicode }
     }
 
     codeMirror.on("focus", function () {
-      if (codeMirror.getValue() == LABELS.NEW_SCRIPT_CONTENT) {
+      if (codeMirror.getValue() == LABELS.NEW_SCRIPT_CONTENT[viewModel.currentScript().language()]) {
         codeMirror.setValue("");
       }
       if (errorWidget != null) {

+ 3 - 2
apps/spark/src/spark/tests.py

@@ -38,7 +38,8 @@ class TestSparkBase(object):
       'script': 'print "spark"',
       'parameters': [],
       'resources': [],
-      'hadoopProperties': []
+      'hadoopProperties': [],
+      'language': 'python'
   }
 
   def setUp(self):
@@ -67,7 +68,7 @@ class TestMock(TestSparkBase):
   def test_save(self):
     attrs = {'user': self.user,}
     attrs.update(TestSparkBase.SCRIPT_ATTRS)
-    #attrs['type'] = json.dumps(TestSparkBase.SCRIPT_ATTRS['type']) # TODO: when support of Scala + Java
+    attrs['language'] = json.dumps(TestSparkBase.SCRIPT_ATTRS['language'])
     attrs['parameters'] = json.dumps(TestSparkBase.SCRIPT_ATTRS['parameters'])
     attrs['resources'] = json.dumps(TestSparkBase.SCRIPT_ATTRS['resources'])
     attrs['hadoopProperties'] = json.dumps(TestSparkBase.SCRIPT_ATTRS['hadoopProperties'])

+ 7 - 2
apps/spark/src/spark/views.py

@@ -72,6 +72,7 @@ def save(request):
     'parameters': json.loads(request.POST.get('parameters')),
     'resources': json.loads(request.POST.get('resources')),
     'hadoopProperties': json.loads(request.POST.get('hadoopProperties')),
+    'language': json.loads(request.POST.get('language')),
   }
   spark_script = create_or_update_script(**attrs)
   spark_script.is_design = True
@@ -116,6 +117,7 @@ def run(request):
     'parameters': json.loads(request.POST.get('parameters')),
     'resources': json.loads(request.POST.get('resources')),
     'hadoopProperties': json.loads(request.POST.get('hadoopProperties')),
+    'language': json.loads(request.POST.get('language')),
     'is_design': False
   }
 
@@ -150,6 +152,7 @@ def copy(request):
   parameters = existing_script_data["parameters"]
   resources = existing_script_data["resources"]
   hadoopProperties = existing_script_data["hadoopProperties"]
+  language = existing_script_data["language"]
 
   script_copy = SparkScript.objects.create()
   script_copy.update_from_dict({
@@ -157,7 +160,8 @@ def copy(request):
       'script': script,
       'parameters': parameters,
       'resources': resources,
-      'hadoopProperties': hadoopProperties
+      'hadoopProperties': hadoopProperties,
+      'language': language
   })
   script_copy.save()
 
@@ -170,7 +174,8 @@ def copy(request):
     'script': script,
     'parameters': parameters,
     'resources': resources,
-    'hadoopProperties': hadoopProperties
+    'hadoopProperties': hadoopProperties,
+    'language': language
   }
 
   return HttpResponse(json.dumps(response), content_type="text/plain")

+ 30 - 20
apps/spark/static/js/spark.ko.js

@@ -31,15 +31,15 @@ var HadoopProperty = function (property) {
 var PigParameter = HadoopProperty;
 
 
-var PigScript = function (pigScript) {
+var SparkScript = function (sparkScript) {
   var self = this;
 
-  self.id = ko.observable(pigScript.id);
-  self.isDesign = ko.observable(pigScript.isDesign);
-  self.name = ko.observable(pigScript.name);
-  self.type = ko.observable(pigScript.type);
-  self.script = ko.observable(pigScript.script);
-  self.scriptSumup = ko.observable(pigScript.script.replace(/\W+/g, ' ').substring(0, 100));
+  self.id = ko.observable(sparkScript.id);
+  self.isDesign = ko.observable(sparkScript.isDesign);
+  self.name = ko.observable(sparkScript.name);
+  self.language = ko.observable(sparkScript.language);
+  self.script = ko.observable(sparkScript.script);
+  self.scriptSumup = ko.observable(sparkScript.script.replace(/\W+/g, ' ').substring(0, 100));
   self.isRunning = ko.observable(false);
   self.selected = ko.observable(false);
   self.watchUrl = ko.observable("");
@@ -54,7 +54,7 @@ var PigScript = function (pigScript) {
   };
 
   self.parameters = ko.observableArray([]);
-  ko.utils.arrayForEach(pigScript.parameters, function (parameter) {
+  ko.utils.arrayForEach(sparkScript.parameters, function (parameter) {
     self.parameters.push(new PigParameter({name: parameter.name, value: parameter.value}));
   });
   self.addParameter = function () {
@@ -71,7 +71,7 @@ var PigScript = function (pigScript) {
   };
 
   self.hadoopProperties = ko.observableArray([]);
-  ko.utils.arrayForEach(pigScript.hadoopProperties, function (property) {
+  ko.utils.arrayForEach(sparkScript.hadoopProperties, function (property) {
     self.hadoopProperties.push(new HadoopProperty({name: property.name, value: property.value}));
   });
   self.addHadoopProperties = function () {
@@ -84,7 +84,7 @@ var PigScript = function (pigScript) {
   };
 
   self.resources = ko.observableArray([]);
-  ko.utils.arrayForEach(pigScript.resources, function (resource) {
+  ko.utils.arrayForEach(sparkScript.resources, function (resource) {
     self.resources.push(new Resource({type: resource.type, value: resource.value}));
   });
   self.addResource = function () {
@@ -96,7 +96,7 @@ var PigScript = function (pigScript) {
     self.updateParentModel();
   };
 
-  self.parentModel = pigScript.parentModel;
+  self.parentModel = sparkScript.parentModel;
   self.updateParentModel = function () {
     if (typeof self.parentModel != "undefined" && self.parentModel != null) {
       self.parentModel.isDirty(true);
@@ -163,15 +163,15 @@ var SparkViewModel = function (props) {
   var _defaultScript = {
     id: -1,
     name: self.LABELS.NEW_SCRIPT_NAME,
-    type: 'python',
-    script: self.LABELS.NEW_SCRIPT_CONTENT,
+    language: 'python',
+    script: self.LABELS.NEW_SCRIPT_CONTENT['python'],
     parameters: self.LABELS.NEW_SCRIPT_PARAMETERS,
     resources: self.LABELS.NEW_SCRIPT_RESOURCES,
     hadoopProperties: self.LABELS.NEW_SCRIPT_HADOOP_PROPERTIES,
     parentModel: self
   };
 
-  self.currentScript = ko.observable(new PigScript(_defaultScript));
+  self.currentScript = ko.observable(new SparkScript(_defaultScript));
   self.loadingScript = null;
   self.currentDeleteType = ko.observable("");
 
@@ -215,7 +215,7 @@ var SparkViewModel = function (props) {
       self.currentScript(_s);
     }
     else {
-      self.currentScript(new PigScript(_defaultScript));
+      self.currentScript(new SparkScript(_defaultScript));
     }
   }
 
@@ -239,14 +239,22 @@ var SparkViewModel = function (props) {
 
   self.newScript = function () {
     self.loadingScript = null;
-    self.currentScript(new PigScript(_defaultScript));
+    self.currentScript(new SparkScript(_defaultScript));
     self.isDirty(false);
     $("#confirmModal").modal("hide");
+    $("#newScriptModal").modal("show");
     $(document).trigger("loadEditor");
     $(document).trigger("showEditor");
     $(document).trigger("clearLogs");
   };
 
+  self.createAndEditScript = function () {
+	self.currentScript().script(LABELS.NEW_SCRIPT_CONTENT[self.currentScript().language()]);
+	$(document).trigger("loadEditor");
+	$("#newScriptModal").modal("hide");
+	$(document).trigger("showEditor");
+  };
+
   self.editScript = function (script) {
     $(document).trigger("showEditor");
   };
@@ -338,7 +346,7 @@ var SparkViewModel = function (props) {
     $.getJSON(self.LIST_SCRIPTS, function (data) {
       self.scripts(ko.utils.arrayMap(data, function (script) {
         script.parentModel = self;
-        return new PigScript(script);
+        return new SparkScript(script);
       }));
       self.filteredScripts(self.scripts());
       $(document).trigger("scriptsRefreshed");
@@ -460,6 +468,7 @@ var SparkViewModel = function (props) {
           parameters: ko.toJSON(script.parameters()),
           resources: ko.toJSON(script.resources()),
           hadoopProperties: ko.toJSON(script.hadoopProperties()),
+          language: ko.toJSON(script.language())
         },
         function (data) {
           self.currentScript().id(data.id);
@@ -481,7 +490,8 @@ var SparkViewModel = function (props) {
           parameters: ko.toJSON(script.parameters()),
           submissionVariables: ko.utils.stringifyJson(self.submissionVariables()),
           resources: ko.toJSON(script.resources()),
-          hadoopProperties: ko.toJSON(script.hadoopProperties())
+          hadoopProperties: ko.toJSON(script.hadoopProperties()),
+          language: ko.toJSON(script.language())
         },
         function (data) {
           if (data.id && self.currentScript().id() != data.id){
@@ -516,7 +526,7 @@ var SparkViewModel = function (props) {
         },
         function (data) {
           data.parentModel = self;
-          self.currentScript(new PigScript(data));
+          self.currentScript(new SparkScript(data));
           $(document).trigger("loadEditor");
           self.updateScripts();
         }, "json");
@@ -524,7 +534,7 @@ var SparkViewModel = function (props) {
 
   function callDelete(ids) {
     if (ids.indexOf(self.currentScript().id()) > -1) {
-      self.currentScript(new PigScript(_defaultScript));
+      self.currentScript(new SparkScript(_defaultScript));
       $(document).trigger("loadEditor");
     }
     $.post(self.DELETE_URL, {

+ 4 - 0
desktop/conf.dist/hue.ini

@@ -609,6 +609,10 @@
 ###########################################################################
 
 [spark]
+  # Address of the Spark master, e.g spark://localhost:7077. If empty use the current configuration.
+  # Can be overriden in the script too.
+  ## spark_master=
+
   # Local path to Spark Home on all the nodes of the cluster.
   ## spark_home=/usr/lib/spark
 

+ 4 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -614,6 +614,10 @@
 ###########################################################################
 
 [spark]
+  # Address of the Spark master, e.g spark://localhost:7077. If empty use the current configuration.
+  # Can be overriden in the script too.
+  ## spark_master=
+
   # Local path to Spark Home on all the nodes of the cluster.
   ## spark_home=/usr/lib/spark
 

+ 362 - 0
desktop/core/static/js/clike.js

@@ -0,0 +1,362 @@
+CodeMirror.defineMode("clike", function(config, parserConfig) {
+  var indentUnit = config.indentUnit,
+      statementIndentUnit = parserConfig.statementIndentUnit || indentUnit,
+      dontAlignCalls = parserConfig.dontAlignCalls,
+      keywords = parserConfig.keywords || {},
+      builtin = parserConfig.builtin || {},
+      blockKeywords = parserConfig.blockKeywords || {},
+      atoms = parserConfig.atoms || {},
+      hooks = parserConfig.hooks || {},
+      multiLineStrings = parserConfig.multiLineStrings;
+  var isOperatorChar = /[+\-*&%=<>!?|\/]/;
+
+  var curPunc;
+
+  function tokenBase(stream, state) {
+    var ch = stream.next();
+    if (hooks[ch]) {
+      var result = hooks[ch](stream, state);
+      if (result !== false) return result;
+    }
+    if (ch == '"' || ch == "'") {
+      state.tokenize = tokenString(ch);
+      return state.tokenize(stream, state);
+    }
+    if (/[\[\]{}\(\),;\:\.]/.test(ch)) {
+      curPunc = ch;
+      return null;
+    }
+    if (/\d/.test(ch)) {
+      stream.eatWhile(/[\w\.]/);
+      return "number";
+    }
+    if (ch == "/") {
+      if (stream.eat("*")) {
+        state.tokenize = tokenComment;
+        return tokenComment(stream, state);
+      }
+      if (stream.eat("/")) {
+        stream.skipToEnd();
+        return "comment";
+      }
+    }
+    if (isOperatorChar.test(ch)) {
+      stream.eatWhile(isOperatorChar);
+      return "operator";
+    }
+    stream.eatWhile(/[\w\$_]/);
+    var cur = stream.current();
+    if (keywords.propertyIsEnumerable(cur)) {
+      if (blockKeywords.propertyIsEnumerable(cur)) curPunc = "newstatement";
+      return "keyword";
+    }
+    if (builtin.propertyIsEnumerable(cur)) {
+      if (blockKeywords.propertyIsEnumerable(cur)) curPunc = "newstatement";
+      return "builtin";
+    }
+    if (atoms.propertyIsEnumerable(cur)) return "atom";
+    return "variable";
+  }
+
+  function tokenString(quote) {
+    return function(stream, state) {
+      var escaped = false, next, end = false;
+      while ((next = stream.next()) != null) {
+        if (next == quote && !escaped) {end = true; break;}
+        escaped = !escaped && next == "\\";
+      }
+      if (end || !(escaped || multiLineStrings))
+        state.tokenize = null;
+      return "string";
+    };
+  }
+
+  function tokenComment(stream, state) {
+    var maybeEnd = false, ch;
+    while (ch = stream.next()) {
+      if (ch == "/" && maybeEnd) {
+        state.tokenize = null;
+        break;
+      }
+      maybeEnd = (ch == "*");
+    }
+    return "comment";
+  }
+
+  function Context(indented, column, type, align, prev) {
+    this.indented = indented;
+    this.column = column;
+    this.type = type;
+    this.align = align;
+    this.prev = prev;
+  }
+  function pushContext(state, col, type) {
+    var indent = state.indented;
+    if (state.context && state.context.type == "statement")
+      indent = state.context.indented;
+    return state.context = new Context(indent, col, type, null, state.context);
+  }
+  function popContext(state) {
+    var t = state.context.type;
+    if (t == ")" || t == "]" || t == "}")
+      state.indented = state.context.indented;
+    return state.context = state.context.prev;
+  }
+
+  // Interface
+
+  return {
+    startState: function(basecolumn) {
+      return {
+        tokenize: null,
+        context: new Context((basecolumn || 0) - indentUnit, 0, "top", false),
+        indented: 0,
+        startOfLine: true
+      };
+    },
+
+    token: function(stream, state) {
+      var ctx = state.context;
+      if (stream.sol()) {
+        if (ctx.align == null) ctx.align = false;
+        state.indented = stream.indentation();
+        state.startOfLine = true;
+      }
+      if (stream.eatSpace()) return null;
+      curPunc = null;
+      var style = (state.tokenize || tokenBase)(stream, state);
+      if (style == "comment" || style == "meta") return style;
+      if (ctx.align == null) ctx.align = true;
+
+      if ((curPunc == ";" || curPunc == ":" || curPunc == ",") && ctx.type == "statement") popContext(state);
+      else if (curPunc == "{") pushContext(state, stream.column(), "}");
+      else if (curPunc == "[") pushContext(state, stream.column(), "]");
+      else if (curPunc == "(") pushContext(state, stream.column(), ")");
+      else if (curPunc == "}") {
+        while (ctx.type == "statement") ctx = popContext(state);
+        if (ctx.type == "}") ctx = popContext(state);
+        while (ctx.type == "statement") ctx = popContext(state);
+      }
+      else if (curPunc == ctx.type) popContext(state);
+      else if (((ctx.type == "}" || ctx.type == "top") && curPunc != ';') || (ctx.type == "statement" && curPunc == "newstatement"))
+        pushContext(state, stream.column(), "statement");
+      state.startOfLine = false;
+      return style;
+    },
+
+    indent: function(state, textAfter) {
+      if (state.tokenize != tokenBase && state.tokenize != null) return CodeMirror.Pass;
+      var ctx = state.context, firstChar = textAfter && textAfter.charAt(0);
+      if (ctx.type == "statement" && firstChar == "}") ctx = ctx.prev;
+      var closing = firstChar == ctx.type;
+      if (ctx.type == "statement") return ctx.indented + (firstChar == "{" ? 0 : statementIndentUnit);
+      else if (ctx.align && (!dontAlignCalls || ctx.type != ")")) return ctx.column + (closing ? 0 : 1);
+      else if (ctx.type == ")" && !closing) return ctx.indented + statementIndentUnit;
+      else return ctx.indented + (closing ? 0 : indentUnit);
+    },
+
+    electricChars: "{}",
+    blockCommentStart: "/*",
+    blockCommentEnd: "*/",
+    lineComment: "//",
+    fold: "brace"
+  };
+});
+
+(function() {
+  function words(str) {
+    var obj = {}, words = str.split(" ");
+    for (var i = 0; i < words.length; ++i) obj[words[i]] = true;
+    return obj;
+  }
+  var cKeywords = "auto if break int case long char register continue return default short do sizeof " +
+    "double static else struct entry switch extern typedef float union for unsigned " +
+    "goto while enum void const signed volatile";
+
+  function cppHook(stream, state) {
+    if (!state.startOfLine) return false;
+    for (;;) {
+      if (stream.skipTo("\\")) {
+        stream.next();
+        if (stream.eol()) {
+          state.tokenize = cppHook;
+          break;
+        }
+      } else {
+        stream.skipToEnd();
+        state.tokenize = null;
+        break;
+      }
+    }
+    return "meta";
+  }
+
+  // C#-style strings where "" escapes a quote.
+  function tokenAtString(stream, state) {
+    var next;
+    while ((next = stream.next()) != null) {
+      if (next == '"' && !stream.eat('"')) {
+        state.tokenize = null;
+        break;
+      }
+    }
+    return "string";
+  }
+
+  function mimes(ms, mode) {
+    for (var i = 0; i < ms.length; ++i) CodeMirror.defineMIME(ms[i], mode);
+  }
+
+  mimes(["text/x-csrc", "text/x-c", "text/x-chdr"], {
+    name: "clike",
+    keywords: words(cKeywords),
+    blockKeywords: words("case do else for if switch while struct"),
+    atoms: words("null"),
+    hooks: {"#": cppHook}
+  });
+  mimes(["text/x-c++src", "text/x-c++hdr"], {
+    name: "clike",
+    keywords: words(cKeywords + " asm dynamic_cast namespace reinterpret_cast try bool explicit new " +
+                    "static_cast typeid catch operator template typename class friend private " +
+                    "this using const_cast inline public throw virtual delete mutable protected " +
+                    "wchar_t"),
+    blockKeywords: words("catch class do else finally for if struct switch try while"),
+    atoms: words("true false null"),
+    hooks: {"#": cppHook}
+  });
+  CodeMirror.defineMIME("text/x-java", {
+    name: "clike",
+    keywords: words("abstract assert boolean break byte case catch char class const continue default " +
+                    "do double else enum extends final finally float for goto if implements import " +
+                    "instanceof int interface long native new package private protected public " +
+                    "return short static strictfp super switch synchronized this throw throws transient " +
+                    "try void volatile while"),
+    blockKeywords: words("catch class do else finally for if switch try while"),
+    atoms: words("true false null"),
+    hooks: {
+      "@": function(stream) {
+        stream.eatWhile(/[\w\$_]/);
+        return "meta";
+      }
+    }
+  });
+  CodeMirror.defineMIME("text/x-csharp", {
+    name: "clike",
+    keywords: words("abstract as base break case catch checked class const continue" +
+                    " default delegate do else enum event explicit extern finally fixed for" +
+                    " foreach goto if implicit in interface internal is lock namespace new" +
+                    " operator out override params private protected public readonly ref return sealed" +
+                    " sizeof stackalloc static struct switch this throw try typeof unchecked" +
+                    " unsafe using virtual void volatile while add alias ascending descending dynamic from get" +
+                    " global group into join let orderby partial remove select set value var yield"),
+    blockKeywords: words("catch class do else finally for foreach if struct switch try while"),
+    builtin: words("Boolean Byte Char DateTime DateTimeOffset Decimal Double" +
+                    " Guid Int16 Int32 Int64 Object SByte Single String TimeSpan UInt16 UInt32" +
+                    " UInt64 bool byte char decimal double short int long object"  +
+                    " sbyte float string ushort uint ulong"),
+    atoms: words("true false null"),
+    hooks: {
+      "@": function(stream, state) {
+        if (stream.eat('"')) {
+          state.tokenize = tokenAtString;
+          return tokenAtString(stream, state);
+        }
+        stream.eatWhile(/[\w\$_]/);
+        return "meta";
+      }
+    }
+  });
+  CodeMirror.defineMIME("text/x-scala", {
+    name: "clike",
+    keywords: words(
+
+      /* scala */
+      "abstract case catch class def do else extends false final finally for forSome if " +
+      "implicit import lazy match new null object override package private protected return " +
+      "sealed super this throw trait try trye type val var while with yield _ : = => <- <: " +
+      "<% >: # @ " +
+
+      /* package scala */
+      "assert assume require print println printf readLine readBoolean readByte readShort " +
+      "readChar readInt readLong readFloat readDouble " +
+
+      "AnyVal App Application Array BufferedIterator BigDecimal BigInt Char Console Either " +
+      "Enumeration Equiv Error Exception Fractional Function IndexedSeq Integral Iterable " +
+      "Iterator List Map Numeric Nil NotNull Option Ordered Ordering PartialFunction PartialOrdering " +
+      "Product Proxy Range Responder Seq Serializable Set Specializable Stream StringBuilder " +
+      "StringContext Symbol Throwable Traversable TraversableOnce Tuple Unit Vector :: #:: " +
+
+      /* package java.lang */
+      "Boolean Byte Character CharSequence Class ClassLoader Cloneable Comparable " +
+      "Compiler Double Exception Float Integer Long Math Number Object Package Pair Process " +
+      "Runtime Runnable SecurityManager Short StackTraceElement StrictMath String " +
+      "StringBuffer System Thread ThreadGroup ThreadLocal Throwable Triple Void"
+
+
+    ),
+    blockKeywords: words("catch class do else finally for forSome if match switch try while"),
+    atoms: words("true false null"),
+    hooks: {
+      "@": function(stream) {
+        stream.eatWhile(/[\w\$_]/);
+        return "meta";
+      }
+    }
+  });
+  mimes(["x-shader/x-vertex", "x-shader/x-fragment"], {
+    name: "clike",
+    keywords: words("float int bool void " +
+                    "vec2 vec3 vec4 ivec2 ivec3 ivec4 bvec2 bvec3 bvec4 " +
+                    "mat2 mat3 mat4 " +
+                    "sampler1D sampler2D sampler3D samplerCube " +
+                    "sampler1DShadow sampler2DShadow" +
+                    "const attribute uniform varying " +
+                    "break continue discard return " +
+                    "for while do if else struct " +
+                    "in out inout"),
+    blockKeywords: words("for while do if else struct"),
+    builtin: words("radians degrees sin cos tan asin acos atan " +
+                    "pow exp log exp2 sqrt inversesqrt " +
+                    "abs sign floor ceil fract mod min max clamp mix step smootstep " +
+                    "length distance dot cross normalize ftransform faceforward " +
+                    "reflect refract matrixCompMult " +
+                    "lessThan lessThanEqual greaterThan greaterThanEqual " +
+                    "equal notEqual any all not " +
+                    "texture1D texture1DProj texture1DLod texture1DProjLod " +
+                    "texture2D texture2DProj texture2DLod texture2DProjLod " +
+                    "texture3D texture3DProj texture3DLod texture3DProjLod " +
+                    "textureCube textureCubeLod " +
+                    "shadow1D shadow2D shadow1DProj shadow2DProj " +
+                    "shadow1DLod shadow2DLod shadow1DProjLod shadow2DProjLod " +
+                    "dFdx dFdy fwidth " +
+                    "noise1 noise2 noise3 noise4"),
+    atoms: words("true false " +
+                "gl_FragColor gl_SecondaryColor gl_Normal gl_Vertex " +
+                "gl_MultiTexCoord0 gl_MultiTexCoord1 gl_MultiTexCoord2 gl_MultiTexCoord3 " +
+                "gl_MultiTexCoord4 gl_MultiTexCoord5 gl_MultiTexCoord6 gl_MultiTexCoord7 " +
+                "gl_FogCoord " +
+                "gl_Position gl_PointSize gl_ClipVertex " +
+                "gl_FrontColor gl_BackColor gl_FrontSecondaryColor gl_BackSecondaryColor " +
+                "gl_TexCoord gl_FogFragCoord " +
+                "gl_FragCoord gl_FrontFacing " +
+                "gl_FragColor gl_FragData gl_FragDepth " +
+                "gl_ModelViewMatrix gl_ProjectionMatrix gl_ModelViewProjectionMatrix " +
+                "gl_TextureMatrix gl_NormalMatrix gl_ModelViewMatrixInverse " +
+                "gl_ProjectionMatrixInverse gl_ModelViewProjectionMatrixInverse " +
+                "gl_TexureMatrixTranspose gl_ModelViewMatrixInverseTranspose " +
+                "gl_ProjectionMatrixInverseTranspose " +
+                "gl_ModelViewProjectionMatrixInverseTranspose " +
+                "gl_TextureMatrixInverseTranspose " +
+                "gl_NormalScale gl_DepthRange gl_ClipPlane " +
+                "gl_Point gl_FrontMaterial gl_BackMaterial gl_LightSource gl_LightModel " +
+                "gl_FrontLightModelProduct gl_BackLightModelProduct " +
+                "gl_TextureColor gl_EyePlaneS gl_EyePlaneT gl_EyePlaneR gl_EyePlaneQ " +
+                "gl_FogParameters " +
+                "gl_MaxLights gl_MaxClipPlanes gl_MaxTextureUnits gl_MaxTextureCoords " +
+                "gl_MaxVertexAttribs gl_MaxVertexUniformComponents gl_MaxVaryingFloats " +
+                "gl_MaxVertexTextureImageUnits gl_MaxTextureImageUnits " +
+                "gl_MaxFragmentUniformComponents gl_MaxCombineTextureImageUnits " +
+                "gl_MaxDrawBuffers"),
+    hooks: {"#": cppHook}
+  });
+}());