Carve out a code path to defer compilation of generated code...

So that we can generate a big module string which can be generated at
build-time and saved to disk. This is important for two reasons:

1. This allows the browser to avoid using scion compiler at runtime.
Compiled SCXML-to-js modules are much smaller runtime dependency over
the wire than the SCION compiler.

2. Node.js --inspect does not work for code evaluated at runtime using
eval or node vm module. See issue nodejs/node#8369.
So to use chrome dev tools and source maps to debug SCXML, it is
currently necessary to use ahead-of-time compilation to generate a
module file as a workaround.

Author: jbeard4

dist/scxml.js

@@ -29,47 +29,52 @@ function stripAttrNsPrefix(attrName){
   return m[1];
 }
 
+function parseJsCode(fnBody, action, docUrl, node, isExpression){
+  //console.log(fnBody, docUrl);
+  var tokens = esprima.tokenize(fnBody, { loc: true });
+  var lastTokenEndLine = 1,
+      lastTokenEndCol = 0;
+  tokens.forEach(function(token, i){
+    var numLinesPadding = token.loc.start.line - lastTokenEndLine,
+        numColsPadding = token.loc.start.column - lastTokenEndCol;
+    var whitespace = (function(){
+      var s = [];
+      for(var i = 0; i < numLinesPadding; i++){
+        s.push('\n');
+      }
+      for(var j = 0; j < numColsPadding ; j++){
+        s.push(' ');
+      }
+      return s.join('');
+    })();
+    if(!(isExpression && i==0)){ 
+      //skip whitespace
+      node.add(whitespace);
+    }
+    var tokenNode = new SourceNode(
+      action.$line + token.loc.start.line,
+      token.loc.start.line === 1 ? action.$column + token.loc.start.column :  token.loc.start.column,
+      docUrl,
+      token.value 
+    );
+    //console.log('tokenNode',tokenNode);
+    node.add(tokenNode);
+    lastTokenEndLine = token.loc.end.line;
+    lastTokenEndCol = token.loc.end.column;
+  });
+}
+
 function generateFnDeclaration(fnName,fnBody,action,docUrl,isExpression,parseGeneratedCodeForSourceMap){
     if(printTrace) console.log('generateFnDeclaration',fnName,fnBody);
 
-    var tokens = esprima.tokenize(fnBody, { loc: true });
-    var lastTokenEndLine = 1,
-        lastTokenEndCol = 0;
     var node = new SourceNode();
     node.add('function ' + fnName + FN_ARGS + '{\n');
     if(isExpression){
       var returnNode = new SourceNode(action.$line + 1, action.$column, docUrl, 'return ');
       node.add(returnNode);
     }
     if(parseGeneratedCodeForSourceMap){
-      tokens.forEach(function(token, i){
-        var numLinesPadding = token.loc.start.line - lastTokenEndLine,
-            numColsPadding = token.loc.start.column - lastTokenEndCol;
-        var whitespace = (function(){
-          var s = [];
-          for(var i = 0; i < numLinesPadding; i++){
-            s.push('\n');
-          }
-          for(var j = 0; j < numColsPadding ; j++){
-            s.push(' ');
-          }
-          return s.join('');
-        })();
-        if(!(isExpression && i==0)){ 
-          //skip whitespace
-          node.add(whitespace);
-        }
-        var tokenNode = new SourceNode(
-          action.$line + token.loc.start.line,
-          token.loc.start.line === 1 ? action.$column + token.loc.start.column :  token.loc.start.column,
-          docUrl,
-          token.value 
-        );
-        //console.log('tokenNode',tokenNode);
-        node.add(tokenNode);
-        lastTokenEndLine = token.loc.end.line;
-        lastTokenEndCol = token.loc.end.column;
-      });
+      parseJsCode(fnBody, action, docUrl, node, isExpression);
     }else{
       node.add( 
         new SourceNode(
@@ -194,16 +199,17 @@ function dumpFunctionDeclarations(fnDecAccumulator){
 
 function dumpHeader(strict){
     var d = new Date();
-    var strictStr = (strict ? "'use strict';\n" : "");
-    return strictStr + '//Generated on ' + d.toLocaleDateString() + ' ' + d.toLocaleTimeString() + ' by the SCION SCXML compiler';
+    return '//Generated on ' + d.toLocaleDateString() + ' ' + d.toLocaleTimeString() + ' by the SCION SCXML compiler\n' + 
+            (strict ? "'use strict';\n" : "");
 }
 
 function generateFactoryFunctionWrapper(o, name, options){
 
     var root = new SourceNode(null,null,null,[
-              o.headerString + '\n' ,
+              //o.headerString + '\n' ,
               '(function (_x,_sessionid,_ioprocessors,In){\n' ,
               '   var _name = \'' + name + '\';\n',
+              new SourceNode(null,null,null,""),    //create a dummy node at index 2, which we might use to inject datamodel and scripts
               o.sendString,
               o.sendIdLocationString,
               o.earlyBindingFnDeclaration,
@@ -218,11 +224,7 @@ function generateFactoryFunctionWrapper(o, name, options){
       ])
     );
 
-    var s = root.toStringWithSourceMap();
-
-    return s.code + '\n' +
-      '//# sourceMappingURL=data:application/json;charset=utf-8;base64,' + 
-      new Buffer(s.map.toString()).toString('base64');
+    return root;
 }
 
 ModuleBuilder.prototype.generateModule = function(){
@@ -420,7 +422,7 @@ ModuleBuilder.prototype.handleExternalActionScript = function(action) {
     if (!this.externalActionScripts.has(action.src)) {
         this.externalActionScripts.add(action.src);
         action.$wrap = function(body) {
-            return generateFnDeclaration(fnName, body, action);
+            return generateFnDeclaration(fnName,body,{$line : 0, $column:0},action.src,false,true);
         }
 
         this.rootState.rootScripts.push(action);
@@ -652,6 +654,7 @@ var actionTags = {
         var shallowArrayName = getVariableNameForShallowCopy(builder);
 
         var forEachContents =
+            (!action.index ? 'var ' + index + ';\n' : '') + 
             'var ' + shallowArrayName + ' = ' + arr + ';\n'+
             'if(Array.isArray(' + shallowArrayName + ')){\n' +
             '    for(' + index + ' = 0; ' + index + ' < ' + shallowArrayName + '.length;' + index + '++){\n' +
@@ -712,54 +715,8 @@ function generateIdlocationGenerator(sendIdAccumulator){
       '}';
 }
 
-
-module.exports = startTraversal;
-
-//for executing directly under node.js
-if(require.main === module){
-    //TODO: clean up command-line interface so that we do not expose unnecessary cruft
-    var usage = 'Usage: $0 [ FILE | - ]';
-    var argv = require('optimist').
-                demand('d').
-                alias('d', 'doc-url').
-                usage(usage).
-                argv;
-    var input = argv._[0];
-    var docUrl = argv.d;
-
-    if(!input){
-      console.error(usage);
-      process.exit(1);
-    } else if(input === '-'){
-
-      //read from stdin or file
-      process.stdin.setEncoding('utf8');
-      process.stdin.resume();
-
-      var jsonString = '';
-      process.stdin.on('data',function(s){
-          jsonString  += s;
-      });
-
-      process.stdin.on('end',go.bind(this,docUrl));
-    } else{
-      var fs = require('fs');
-      jsonString = fs.readFileSync(input,'utf8');
-      go(docUrl);
-    }
-}
-
-function go(docUrl){
-    if (!docUrl) {
-        docUrl = '';
-    }
-
-    startTraversal(docUrl, JSON.parse(jsonString), {debug: true}).then(
-        function resolved(jsModule) {
-            console.log(jsModule.module);
-        },
-        function rejected(err) {
-            console.error(err);
-        }
-    );
-}
+module.exports = { 
+  startTraversal : startTraversal, 
+  parseJsCode : parseJsCode,
+  dumpHeader : dumpHeader
+};

lib/compiler/scjson-to-module.js

@@ -332,10 +332,3 @@ function transform(xmlString){
 }
 
 module.exports = transform;
-
-//for executing diretly under node.js
-if(require.main === module){
-    //TODO: allow reading from stdin directly
-    //TODO: use saxjs's support for streaming API.
-    console.log(JSON.stringify(transform(require('fs').readFileSync(process.argv[2],'utf8')),4,4));
-}

lib/compiler/scxml-to-scjson.js

@@ -21,6 +21,7 @@ var compilerInternals = require('./compiler-internals'),
     platform = require('./platform-bootstrap/node/platform'),
     fs = require('fs'),
     vm = require('vm'),
+    SourceNode = require('source-map').SourceNode,
     assert = require('assert');
 
 var scxmlToScjson = compilerInternals.scxmlToScjson, 
@@ -55,12 +56,15 @@ function documentStringToModel(url,docString,cb,hostContext){
 function fetchScript(scriptInfo, hostContext) {
     return new Promise(function(resolve, reject) {
 
-        platform.getScriptFromUrl(scriptInfo.src,
+        platform.getScriptFromUrl(scriptInfo,
             function(err, compiledScript, mimeType) {
                 if (err) {
                     reject(err);
                 } else {
-                    scriptInfo.compiled = compiledScript;
+                    if(!hostContext.deferCompilation){
+                        var content = scriptInfo.$wrap ? scriptInfo.$wrap(scriptInfo.content) : scriptInfo.content;
+                        scriptInfo.compiled = new vm.Script(content, {filename: scriptInfo.src});
+                    }
                     resolve(scriptInfo);
                 }
             },
@@ -199,16 +203,49 @@ SCModel.prototype.prepare = function(cb, executionContext, hostContext) {
     const self = this;
     Promise.all(scriptPromises).then(function resolved(scripts) {
         try {
-            if (self.datamodel) {
-                self.datamodel.runInContext(executionContext);
-            }
+            if(hostContext.deferCompilation){
+                var rootNode = new SourceNode(null, null, null);
 
-            for (let i = 0; i < scriptCount; i++) {
-                self.rootScripts[i].compiled.runInContext(executionContext);
-            }
+                rootNode.add(scjsonToModule.dumpHeader(true));
+
+                //rootNode.add('module.exports = ');
+                var injectionNode = self.module.children[2];
+
+                if(self.datamodel) {
+                  injectionNode.add(self.datamodel);
+                  injectionNode.add('\n');
+                }
+
+                self.rootScripts.forEach(function(rootScript){
+                  if(rootScript.$wrap){
+                    injectionNode.add(rootScript.$wrap(rootScript.content));
+                  }else{
+                    scjsonToModule.parseJsCode(rootScript.content, {$line : 0, $column : 0}, rootScript.src, injectionNode, false)
+                  }
+                  injectionNode.add('\n');
+                })
+
+                rootNode.add(self.module);
+
+                var s = rootNode.toStringWithSourceMap();
+                var generatedCode = s.code + '\n' +
+                      '//# sourceMappingURL=data:application/json;charset=utf-8;base64,' + 
+                      new Buffer(s.map.toString()).toString('base64')
+                //console.log(generatedCode);
+                var fn = vm.runInContext(generatedCode, executionContext);
+                cb(null,fn);
+            } else {
+                if (self.datamodel) {
+                    self.datamodel.runInContext(executionContext);
+                }
+
+                for (let i = 0; i < scriptCount; i++) {
+                    self.rootScripts[i].compiled.runInContext(executionContext);
+                }
 
-            let modelFn = self.module.runInContext(executionContext);
-            cb(undefined, modelFn);
+                let modelFn = self.module.runInContext(executionContext);
+                cb(undefined, modelFn);
+            }
         } catch (e) {
             cb(e);
         }
@@ -225,12 +262,21 @@ function createModule(url,scJson,hostContext,cb){
         }
     }
 
-    scjsonToModule(url, scJson, hostContext).then(function resolved(rawModule) {
+    scjsonToModule.startTraversal(url, scJson, hostContext).then(function resolved(rawModule) {
         if (platform.debug && rawModule.name) {
             fs.writeFileSync('/var/tmp/' + rawModule.name + '.scion', rawModule.module);
         }
 
-        compileModule(url, rawModule, hostContext, cb);
+        if(hostContext.deferCompilation){
+          cb(null, new SCModel(
+              rawModule.name,
+              rawModule.datamodel,
+              rawModule.rootScripts,
+              rawModule.module
+          ));
+        } else {
+          compileModule(url, rawModule, hostContext, cb);
+        }
     }, function rejected(err) {
         cb(err);
     });

lib/runtime/document-string-to-model.js

@@ -84,3 +84,16 @@ module.exports = {
     },
     scion : require('scion-core')
 };
+
+
+if(require.main === module){
+  var file = process.argv[2];
+  var options = {deferCompilation : true};
+  pathToModel(file, function(err, model){
+    if(err) throw err;
+    model.prepare(function(err, fnModel){
+      if(err) throw err;
+      console.log(fnModel.toString());
+    }, options);
+  }, options); 
+}

lib/runtime/facade.js

@@ -40,21 +40,16 @@ module.exports = {
 
     getResourceFromUrl : get.getResource,
 
-    getScriptFromUrl: function(url, cb, context, scriptInfo) {
-        get.getResource(url, function(err, content) {
+    getScriptFromUrl: function(script, cb, context, scriptInfo) {
+        get.getResource(script.src, function(err, content) {
             if (err) {
                 cb(err);
                 return;
             }
 
-            if (typeof scriptInfo.$wrap === 'function') {
-                content = scriptInfo.$wrap(content);
-            }
-
-            var options = Object.assign({filename: url}, scriptInfo);
             try {
-                var script = new vm.Script(content, options);
-                cb(undefined, script);
+                script.content = content;
+                cb(undefined, content);
             } catch (e) {
                 cb(e);
             }

lib/runtime/platform-bootstrap/node/platform.js

@@ -38,7 +38,7 @@ http.createServer(function (req, res) {
                         try {
                             model.prepare(function(err, fnModel) {
                                 if (err) {
-                                    console.error('model preparation error: ' + err);
+                                    console.error('model preparation error: ' + err, err.stack);
                                     res.writeHead(500, {'Content-Type': 'text/plain'});
                                     res.end(err.message);
                                     return;
@@ -60,15 +60,15 @@ http.createServer(function (req, res) {
 
                                 // TODO: timeout should be kicked off before fetch/compilation/preparation
                                 timeouts[sessionToken] = setTimeout(function(){cleanUp(sessionToken);},timeoutMs);  
-                            });
+                            }, {deferCompilation : true});
                         } catch(e) {
                           console.log(e.stack);
                           console.log(e);
                           res.writeHead(500, {'Content-Type': 'text/plain'});
                           res.end(e.message);
                         }
                     }
-                });
+                }, null, {deferCompilation : true});
 
             }else if(reqJson.event && (typeof reqJson.sessionToken === "number")){
                 console.log("sending event to statechart",reqJson.event);