[react-packager] Rewrite dependency graph (support node_modules, speed, fix bugs etc)

Summary:
@public
Fixes facebook#773, facebook#1055
The resolver was getting a bit unwieldy because a lot has changed since the initial writing (porting node-haste).
This also splits up a large complex file into the following:

* Makes use of classes: Module, AssetModule, Package, and AssetModule_DEPRECATED (`image!` modules)
* DependencyGraph is lazy for everything that isn't haste modules and packages (need to read ahead of time)
* Lazy makes it fast, easier to reason about, and easier to add new loaders
* Has a centralized filesystem wrapper: fast-fs (ffs)
* ffs is async and lazy for any read operation and sync for directory/file lookup which makes it fast
* we can easily drop in different adapters for ffs to be able to build up the tree: watchman, git ls-files, etc
* use es6 for classes and easier to read promise-based code

Follow up diffs will include:
* Using new types (Module, AssetModule etc) in the rest of the codebase (currently we convert to plain object which is a bit of a hack)
* using watchman to build up the fs
* some caching at the object creation level (we are recreating Modules and Packages many times, we can cache them)
* A plugin system for loaders (e.g. @tadeuzagallo wants to add a native module loader)

Test Plan:
* ./runJestTests.sh react-packager
* ./runJestTests.sh PackagerIntegration
* Export open source and run the e2e test
* reset cache
* ./fbrnios.sh run and click around

Author: Amjad Masad

Libraries/JavaScriptAppEngine/Initialization/SourceMap.js

@@ -1,3 +1,4 @@
+
 /**
  * Copyright (c) 2015-present, Facebook, Inc.
  * All rights reserved.
@@ -8,6 +9,7 @@
  *
  * @providesModule SourceMap
  * @generated
+ * @extern
  *
  * This module was generated from `node_modules/source-map` by running
  *

packager/blacklist.js

@@ -11,7 +11,6 @@
 // Don't forget to everything listed here to `testConfig.json`
 // modulePathIgnorePatterns.
 var sharedBlacklist = [
-  __dirname,
   'website',
   'node_modules/react-tools/src/utils/ImmutableObject.js',
   'node_modules/react-tools/src/core/ReactInstanceHandles.js',

packager/react-packager/.babelrc

@@ -0,0 +1,24 @@
+// Keep in sync with packager/transformer.js
+{
+  "retainLines": true,
+  "compact": true,
+  "comments": false,
+  "whitelist": [
+    "es6.arrowFunctions",
+    "es6.blockScoping",
+    // This is the only place where we differ from transformer.js
+    "es6.constants",
+    "es6.classes",
+    "es6.destructuring",
+    "es6.parameters.rest",
+    "es6.properties.computed",
+    "es6.properties.shorthand",
+    "es6.spread",
+    "es6.templateLiterals",
+    "es7.trailingFunctionCommas",
+    "es7.objectRestSpread",
+    "flow",
+    "react"
+  ],
+  "sourceMaps": false
+}

packager/react-packager/index.js

@@ -8,6 +8,10 @@
  */
 'use strict';
 
+require('babel/register')({
+  only: /react-packager\/src/
+});
+
 useGracefulFs();
 
 var Activity = require('./src/Activity');

packager/react-packager/src/AssetServer/index.js

@@ -15,7 +15,7 @@ var Promise = require('bluebird');
 var fs = require('fs');
 var crypto = require('crypto');
 
-var lstat = Promise.promisify(fs.lstat);
+var stat = Promise.promisify(fs.stat);
 var readDir = Promise.promisify(fs.readdir);
 var readFile = Promise.promisify(fs.readFile);
 
@@ -98,14 +98,14 @@ AssetServer.prototype.getAssetData = function(assetPath) {
 
     return Promise.all(
       record.files.map(function(file) {
-        return lstat(file);
+        return stat(file);
       })
     );
   }).then(function(stats) {
     var hash = crypto.createHash('md5');
 
-    stats.forEach(function(stat) {
-      hash.update(stat.mtime.getTime().toString());
+    stats.forEach(function(fstat) {
+      hash.update(fstat.mtime.getTime().toString());
     });
 
     data.hash = hash.digest('hex');
@@ -117,18 +117,18 @@ function findRoot(roots, dir) {
   return Promise.some(
     roots.map(function(root) {
       var absPath = path.join(root, dir);
-      return lstat(absPath).then(function(stat) {
-        if (!stat.isDirectory()) {
+      return stat(absPath).then(function(fstat) {
+        if (!fstat.isDirectory()) {
           throw new Error('Looking for dirs');
         }
-        stat._path = absPath;
-        return stat;
+        fstat._path = absPath;
+        return fstat;
       });
     }),
     1
   ).spread(
-    function(stat) {
-      return stat._path;
+    function(fstat) {
+      return fstat._path;
     }
   );
 }

packager/react-packager/src/DependencyResolver/AssetModule.js

@@ -0,0 +1,46 @@
+'use strict';
+
+const Module = require('./Module');
+const Promise = require('bluebird');
+const getAssetDataFromName = require('../lib/getAssetDataFromName');
+
+class AssetModule extends Module {
+
+  isHaste() {
+    return Promise.resolve(false);
+  }
+
+  getDependencies() {
+    return Promise.resolve([]);
+  }
+
+  _read() {
+    return Promise.resolve({});
+  }
+
+  getName() {
+    return super.getName().then(id => {
+      const {name, type} = getAssetDataFromName(this.path);
+      return id.replace(/\/[^\/]+$/, `/${name}.${type}`);
+    });
+  }
+
+  getPlainObject() {
+    return this.getName().then(name => this.addReference({
+      path: this.path,
+      isJSON: false,
+      isAsset: true,
+      isAsset_DEPRECATED: false,
+      isPolyfill: false,
+      resolution: getAssetDataFromName(this.path).resolution,
+      id: name,
+      dependencies: [],
+    }));
+  }
+
+  hash() {
+    return `AssetModule : ${this.path}`;
+  }
+}
+
+module.exports = AssetModule;

packager/react-packager/src/DependencyResolver/AssetModule_DEPRECATED.js

@@ -0,0 +1,40 @@
+'use strict';
+
+const Module = require('./Module');
+const Promise = require('bluebird');
+const getAssetDataFromName = require('../lib/getAssetDataFromName');
+
+class AssetModule_DEPRECATED extends Module {
+  isHaste() {
+    return Promise.resolve(false);
+  }
+
+  getName() {
+    return Promise.resolve(this.name);
+  }
+
+  getDependencies() {
+    return Promise.resolve([]);
+  }
+
+  getPlainObject() {
+    const {name, resolution} = getAssetDataFromName(this.path);
+
+    return Promise.resolve(this.addReference({
+      path: this.path,
+      id: `image!${name}`,
+      resolution,
+      isAsset_DEPRECATED: true,
+      dependencies: [],
+      isJSON: false,
+      isPolyfill: false,
+      isAsset: false,
+    }));
+  }
+
+  hash() {
+    return `AssetModule_DEPRECATED : ${this.path}`;
+  }
+}
+
+module.exports = AssetModule_DEPRECATED;