[wasm] Enable execution in node

tfjs-backend-wasm/cloudbuild.yml

@@ -13,23 +13,31 @@ steps:
   args: ['install']
   waitFor: ['yarn-common']
 
+# Build core from master.
+- name: 'node:10'
+  dir: 'tfjs-backend-wasm'
+  id: 'build-core'
+  entrypoint: 'yarn'
+  args: ['build-core']
+  waitFor: ['yarn-common']
+
 # Run lint.
 - name: 'node:10'
   dir: 'tfjs-backend-wasm'
   entrypoint: 'yarn'
   id: 'lint'
   args: ['lint']
-  waitFor: ['yarn']
+  waitFor: ['build-core']
 
 # Build the project.
 - name: 'node:10'
   dir: 'tfjs-backend-wasm'
   entrypoint: 'yarn'
   id: 'build'
   args: ['build-ci']
-  waitFor: ['yarn']
+  waitFor: ['build-core']
 
-# Run browser tests.
+# Run tests in browser.
 - name: 'node:10'
   dir: 'tfjs-backend-wasm'
   entrypoint: 'yarn'
@@ -39,6 +47,14 @@ steps:
   env: ['BROWSERSTACK_USERNAME=deeplearnjs1']
   secretEnv: ['BROWSERSTACK_KEY']
 
+# Run tests in node.
+- name: 'node:10'
+  dir: 'tfjs-backend-wasm'
+  entrypoint: 'yarn'
+  id: 'test-node'
+  args: ['test-node']
+  waitFor: ['build']
+
 # Run C++ tests.
 - name: 'node:10'
   dir: 'tfjs-backend-wasm'

tfjs-backend-wasm/karma.conf.js

@@ -54,6 +54,7 @@ module.exports = function(config) {
       // Import the rest of the sources.
       {pattern: 'src/**/*.ts'},
     ],
+    exclude: ['src/test_node.ts'],
     preprocessors: {
       'wasm-out/**/*.js': ['karma-typescript'],
       '**/*.ts': ['karma-typescript']

tfjs-backend-wasm/package.json

@@ -8,15 +8,16 @@
   "unpkg": "dist/tf-wasm.min.js",
   "jsdelivr": "dist/tf-wasm.min.js",
   "scripts": {
-    "preinstall": "cd ../tfjs-core && yarn && yarn build",
-    "build": "rimraf dist/ && yarn link-core-master && tsc && ./scripts/build-wasm.sh",
+    "build-core": "yarn link-core-master && cd ../tfjs-core && yarn && yarn build",
+    "build": "yarn build-core && rimraf dist/ && tsc && ./scripts/build-wasm.sh",
     "build-ci": "./scripts/build-ci.sh",
     "build-npm": "./scripts/build-npm.sh",
     "clean": "rimraf dist/ && bazel clean --expunge",
     "cpplint": "./scripts/cpplint.js",
     "lint": "tslint -p . -t verbose && yarn cpplint",
     "link-core-master": "./scripts/link-core-master.js",
     "test": "yarn link-core-master && ./scripts/build-wasm.sh && karma start",
+    "test-node": "ts-node src/test_node.ts",
     "test-bundle-size": "./scripts/test-bundle-size.js",
     "test-cc": "bazel test //src/cc:cc_tests --test_output=all",
     "test-browser-ci": "karma start --singleRun --browsers=bs_chrome_mac"
@@ -29,7 +30,8 @@
     "@tensorflow/tfjs-core": "link:../tfjs-core",
     "@types/emscripten": "~0.0.34",
     "@types/jasmine": "~2.8.6",
-    "clang-format": "^1.2.4",
+    "clang-format": "~1.2.4",
+    "jasmine": "~3.1.0",
     "jasmine-core": "~3.1.0",
     "karma": "~4.0.0",
     "karma-browserstack-launcher": "~1.4.0",
@@ -41,10 +43,11 @@
     "rollup": "~1.26.3",
     "rollup-plugin-commonjs": "~10.1.0",
     "rollup-plugin-node-resolve": "~5.2.0",
-    "rollup-plugin-terser": "^5.1.1",
+    "rollup-plugin-terser": "~5.1.1",
     "rollup-plugin-typescript2": "~0.25.2",
-    "tslint": "^5.20.0",
-    "tslint-no-circular-imports": "^0.7.0",
+    "ts-node": "~8.5.4",
+    "tslint": "~5.20.0",
+    "tslint-no-circular-imports": "~0.7.0",
     "typescript": "3.5.3",
     "yalc": "~1.0.0-pre.21"
   },

tfjs-backend-wasm/rollup.config.js

@@ -78,7 +78,7 @@ module.exports = cmdOptions => {
     bundles.push(config({
       output: {
         format: 'umd',
-        name: 'tf',
+        name: 'tf.wasm',
         extend: true,
         file: 'dist/tf-backend-wasm.js',
       }
@@ -90,7 +90,7 @@ module.exports = cmdOptions => {
     plugins: [terser({output: {preamble: PREAMBLE}})],
     output: {
       format: 'umd',
-      name: 'tf',
+      name: 'tf.wasm',
       extend: true,
       file: 'dist/tf-backend-wasm.min.js',
     },

tfjs-backend-wasm/src/cc/BUILD

@@ -18,7 +18,6 @@ cc_binary(
         "-s EXIT_RUNTIME=0",
         "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]'",
         "-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"cwrap\"]'",
-        "-s ENVIRONMENT=web",
         "-s MODULARIZE=1",
         "-s EXPORT_NAME=WasmBackendModule",
         "-s MALLOC=emmalloc",

tfjs-backend-wasm/src/test_node.ts

@@ -0,0 +1,40 @@
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+// Register the wasm backend.
+import './index';
+
+// tslint:disable-next-line: no-imports-from-dist
+import {setTestEnvs} from '@tensorflow/tfjs-core/dist/jasmine_util';
+import jasmine from 'jasmine';
+
+Error.stackTraceLimit = Infinity;
+
+process.on('unhandledRejection', e => {
+  throw e;
+});
+
+setTestEnvs([{name: 'wasm-node', backendName: 'wasm', isDataSync: true}]);
+
+const runner = new jasmine({});
+runner.loadConfig({spec_files: ['src/**/*_test.ts'], random: false});
+
+if (process.env.JASMINE_SEED) {
+  runner.seed(process.env.JASMINE_SEED);
+}
+
+runner.execute();

tfjs-backend-wasm/tsconfig.json

@@ -7,6 +7,7 @@
     "node_modules/"
   ],
   "compilerOptions": {
+    "esModuleInterop": true,
     "outDir": "./dist"
   }
 }

tfjs-backend-wasm/yarn.lock

@@ -200,6 +200,11 @@ are-we-there-yet@~1.1.2:
     delegates "^1.0.0"
     readable-stream "^2.0.6"
 
+arg@^4.1.0:
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.2.tgz#e70c90579e02c63d80e3ad4e31d8bfdb8bd50064"
+  integrity sha512-+ytCkGcBtHZ3V2r2Z06AncYO8jz46UEamcspGoU8lHcEbpn6J77QK0vdWvChsclg/tM5XIJC5tnjmPp7Eq6Obg==
+
 argparse@^1.0.7:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
@@ -647,7 +652,7 @@ cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3:
     inherits "^2.0.1"
     safe-buffer "^5.0.1"
 
-clang-format@^1.2.4:
+clang-format@~1.2.4:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/clang-format/-/clang-format-1.2.4.tgz#4bb4b0a98180428deb093cf20982e9fc1af20b6c"
   integrity sha512-sw+nrGUp3hvmANd1qF8vZPuezSYQAiXgGBiEtkXTtJnnu6b00fCqkkDIsnRKrNgg4nv6NYZE92ejvOMIXZoejw==
@@ -1429,6 +1434,18 @@ glob@^7.0.0, glob@^7.1.1, glob@^7.1.3, glob@^7.1.4:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
+glob@^7.0.6:
+  version "7.1.6"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6"
+  integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==
+  dependencies:
+    fs.realpath "^1.0.0"
+    inflight "^1.0.4"
+    inherits "2"
+    minimatch "^3.0.4"
+    once "^1.3.0"
+    path-is-absolute "^1.0.0"
+
 graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0:
   version "4.2.2"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.2.tgz#6f0952605d0140c1cfdb138ed005775b92d67b02"
@@ -1850,6 +1867,14 @@ jasmine-core@~3.1.0:
   resolved "https://registry.yarnpkg.com/jasmine-core/-/jasmine-core-3.1.0.tgz#a4785e135d5df65024dfc9224953df585bd2766c"
   integrity sha1-pHheE11d9lAk38kiSVPfWFvSdmw=
 
+jasmine@~3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/jasmine/-/jasmine-3.1.0.tgz#2bd59fd7ec6ec0e8acb64e09f45a68ed2ad1952a"
+  integrity sha1-K9Wf1+xuwOistk4J9Fpo7SrRlSo=
+  dependencies:
+    glob "^7.0.6"
+    jasmine-core "~3.1.0"
+
 jest-worker@^24.6.0:
   version "24.9.0"
   resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-24.9.0.tgz#5dbfdb5b2d322e98567898238a9697bcce67b3e5"
@@ -2114,6 +2139,11 @@ make-dir@^3.0.0:
   dependencies:
     semver "^6.0.0"
 
+make-error@^1.1.1:
+  version "1.3.5"
+  resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.5.tgz#efe4e81f6db28cadd605c70f29c831b58ef776c8"
+  integrity sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==
+
 map-cache@^0.2.2:
   version "0.2.2"
   resolved "https://registry.yarnpkg.com/map-cache/-/map-cache-0.2.2.tgz#c32abd0bd6525d9b051645bb4f26ac5dc98a0dbf"
@@ -2966,7 +2996,7 @@ rollup-plugin-node-resolve@~5.2.0:
     resolve "^1.11.1"
     rollup-pluginutils "^2.8.1"
 
-rollup-plugin-terser@^5.1.1:
+rollup-plugin-terser@~5.1.1:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/rollup-plugin-terser/-/rollup-plugin-terser-5.1.2.tgz#3e41256205cb75f196fc70d4634227d1002c255c"
   integrity sha512-sWKBCOS+vUkRtHtEiJPAf+WnBqk/C402fBD9AVHxSIXMqjsY7MnYWKYEUqGixtr0c8+1DjzUEPlNgOYQPVrS1g==
@@ -3183,6 +3213,14 @@ source-map-resolve@^0.5.0:
     source-map-url "^0.4.0"
     urix "^0.1.0"
 
+source-map-support@^0.5.6:
+  version "0.5.16"
+  resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.16.tgz#0ae069e7fe3ba7538c64c98515e35339eac5a042"
+  integrity sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ==
+  dependencies:
+    buffer-from "^1.0.0"
+    source-map "^0.6.0"
+
 source-map-support@~0.5.12:
   version "0.5.13"
   resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.13.tgz#31b24a9c2e73c2de85066c0feb7d44767ed52932"
@@ -3477,20 +3515,31 @@ trim-newlines@^1.0.0:
   resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613"
   integrity sha1-WIeWa7WCpFA6QetST301ARgVphM=
 
+ts-node@~8.5.4:
+  version "8.5.4"
+  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-8.5.4.tgz#a152add11fa19c221d0b48962c210cf467262ab2"
+  integrity sha512-izbVCRV68EasEPQ8MSIGBNK9dc/4sYJJKYA+IarMQct1RtEot6Xp0bXuClsbUSnKpg50ho+aOAx8en5c+y4OFw==
+  dependencies:
+    arg "^4.1.0"
+    diff "^4.0.1"
+    make-error "^1.1.1"
+    source-map-support "^0.5.6"
+    yn "^3.0.0"
+
 [email protected], tslib@^1.8.0, tslib@^1.8.1:
   version "1.10.0"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.10.0.tgz#c3c19f95973fb0a62973fb09d90d961ee43e5c8a"
   integrity sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==
 
-tslint-no-circular-imports@^0.7.0:
+tslint-no-circular-imports@~0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/tslint-no-circular-imports/-/tslint-no-circular-imports-0.7.0.tgz#9df0a15654d66b172e0b7843eed073fa5ae99b5f"
   integrity sha512-k3wxpeMC4ef40UbpfBVHEHIzKfNZq5/SCtAO1YjGsaNTklo+K53/TWLrym+poA65RJFDiYgYNWvkeIIkJNA0Vw==
 
-tslint@^5.20.0:
-  version "5.20.0"
-  resolved "https://registry.yarnpkg.com/tslint/-/tslint-5.20.0.tgz#fac93bfa79568a5a24e7be9cdde5e02b02d00ec1"
-  integrity sha512-2vqIvkMHbnx8acMogAERQ/IuINOq6DFqgF8/VDvhEkBqQh/x6SP0Y+OHnKth9/ZcHQSroOZwUQSN18v8KKF0/g==
+tslint@~5.20.0:
+  version "5.20.1"
+  resolved "https://registry.yarnpkg.com/tslint/-/tslint-5.20.1.tgz#e401e8aeda0152bc44dd07e614034f3f80c67b7d"
+  integrity sha512-EcMxhzCFt8k+/UP5r8waCf/lzmeSyVlqxqMEDQE7rWYiQky8KpIBz1JAoYXfROHrPZ1XXd43q8yQnULOLiBRQg==
   dependencies:
     "@babel/code-frame" "^7.0.0"
     builtin-modules "^1.1.1"
@@ -3799,3 +3848,8 @@ [email protected]:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/yeast/-/yeast-0.1.2.tgz#008e06d8094320c372dbc2f8ed76a0ca6c8ac419"
   integrity sha1-AI4G2AlDIMNy28L47XagymyKxBk=
+
+yn@^3.0.0:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"
+  integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==

tfjs-core/package.json

@@ -21,7 +21,7 @@
   "devDependencies": {
     "@bazel/bazel": "^0.24.0",
     "@bazel/typescript": "^0.27.8",
-    "@types/jasmine": "~2.5.53",
+    "@types/jasmine": "~3.0.0",
     "@types/node": "~9.6.0",
     "@types/node-fetch": "~2.1.2",
     "browserify": "~16.2.3",

tfjs-core/src/backends/cpu/backend_cpu.ts

@@ -103,7 +103,7 @@ export class MathBackendCPU extends KernelBackend {
             'Then call require(\'@tensorflow/tfjs-node\'); (-gpu ' +
             'suffix for CUDA) at the start of your program. ' +
             'Visit https://github.com/tensorflow/tfjs-node for more details.' +
-            '\n============================\n');
+            '\n============================');
       }
     }
     const dataId = {};

tfjs-core/src/backends/cpu/backend_cpu_test.ts

@@ -135,7 +135,7 @@ describeWithFlags('memory cpu', CPU_ENVS, () => {
   });
 });
 
-describeWithFlags('CPU backend has sync init', {}, () => {
+describeWithFlags('CPU backend has sync init', CPU_ENVS, () => {
   it('can do matmul without waiting for ready', async () => {
     tf.registerBackend('my-cpu', () => {
       return new MathBackendCPU();
@@ -149,3 +149,19 @@ describeWithFlags('CPU backend has sync init', {}, () => {
     tf.removeBackend('my-cpu');
   });
 });
+
+// NOTE: This describe is purposefully not a describeWithFlags so that we
+// test tensor allocation where no scopes have been created. The backend
+// here must be set to CPU because we cannot allocate GPU tensors outside
+// a describeWithFlags because the default webgl backend and the test
+// backends share a WebGLContext. When backends get registered, global
+// WebGL state is initialized, which causes the two backends to step on
+// each other and get in a bad state.
+describe('Memory allocation outside a test scope', () => {
+  it('constructing a tensor works', async () => {
+    tf.setBackend('cpu');
+    const a = tf.tensor1d([1, 2, 3]);
+    expectArraysClose(await a.data(), [1, 2, 3]);
+    a.dispose();
+  });
+});