From 7f077710a3d6995c5af0de2028ad6eb8b019ef84 Mon Sep 17 00:00:00 2001 From: dcode Date: Thu, 22 Oct 2020 17:23:36 +0200 Subject: [PATCH 1/5] Make RTrace ESM by default, with UMD fallback --- lib/rtrace/index.js | 14 +- lib/rtrace/package.json | 21 +- lib/rtrace/umd/index.d.ts | 1 + lib/rtrace/umd/index.js | 384 ++++++++++++++++++++++++++++++++++++ lib/rtrace/umd/package.json | 4 + package.json | 15 +- tests/bootstrap/index.ts | 6 +- tests/compiler.js | 2 +- 8 files changed, 431 insertions(+), 16 deletions(-) create mode 100644 lib/rtrace/umd/index.d.ts create mode 100644 lib/rtrace/umd/index.js create mode 100644 lib/rtrace/umd/package.json diff --git a/lib/rtrace/index.js b/lib/rtrace/index.js index 788b5d2dac..f28dc7fbfc 100644 --- a/lib/rtrace/index.js +++ b/lib/rtrace/index.js @@ -26,7 +26,7 @@ function isTLSF(stack) { return stack[0].startsWith(" at ~lib/rt/tlsf/"); } -class Rtrace { +export class Rtrace { constructor(options) { this.options = options || {}; @@ -146,12 +146,12 @@ class Rtrace { var mmInfo = header[0]; var gcInfo = header[1]; var gcInfo2 = header[2]; - const mmTags = [ // 0│L│F + const mmTags = [ [], ["FREE"], ["LEFTFREE"], ["FREE", "LEFTFREE"] - ]; + ]; // 2=LEFTFREE, 1=FREE const gcColor = [ "BLACK", "GRAY", @@ -208,7 +208,7 @@ class Rtrace { this.markShadow(info); this.refCounts.set(ptr, 0); this.blocks.set(ptr, Object.assign(info, { - allocStack: trimStacktrace(new Error().stack, /* onalloc */ 1) + allocStack: trimStacktrace(new Error().stack, 1) // strip onalloc })); } } @@ -264,7 +264,7 @@ class Rtrace { this.refCounts.delete(ptr); this.unmarkShadow(info); let block = this.blocks.get(ptr); - block.freeStack = trimStacktrace(new Error().stack, /* onfree */ 1); + block.freeStack = trimStacktrace(new Error().stack, 1); // strip onfree } } @@ -343,4 +343,6 @@ class Rtrace { } } -exports.Rtrace = Rtrace; +export default { + Rtrace +}; diff --git a/lib/rtrace/package.json b/lib/rtrace/package.json index c46bf3c052..f258fd3faf 100644 --- a/lib/rtrace/package.json +++ b/lib/rtrace/package.json @@ -1,7 +1,24 @@ { "name": "@assemblyscript/rtrace", - "types": "index.d.ts", "version": "0.2.0", "license": "Apache-2.0", - "main": "index.js" + "type": "module", + "main": "index.js", + "types": "index.d.ts", + "exports": { + "import": "./index.js", + "require": "./umd/index.js" + }, + "scripts": { + "build": "npx esm2umd rtrace index.js > umd/index.js" + }, + "files": [ + "index.d.ts", + "index.js", + "package.json", + "umd/index.d.ts", + "umd/index.js", + "umd/package.json", + "README.md" + ] } diff --git a/lib/rtrace/umd/index.d.ts b/lib/rtrace/umd/index.d.ts new file mode 100644 index 0000000000..a940eccbc8 --- /dev/null +++ b/lib/rtrace/umd/index.d.ts @@ -0,0 +1 @@ +export * from ".."; diff --git a/lib/rtrace/umd/index.js b/lib/rtrace/umd/index.js new file mode 100644 index 0000000000..5bdbaaa7e0 --- /dev/null +++ b/lib/rtrace/umd/index.js @@ -0,0 +1,384 @@ +// GENERATED FILE. DO NOT EDIT. +var rtrace = (function(exports) { + "use strict"; + + Object.defineProperty(exports, "__esModule", { + value: true + }); + exports.default = exports.Rtrace = void 0; + // WebAssembly pages are 65536 kb + const PAGE_SIZE_BITS = 16; + const PAGE_SIZE = 1 << PAGE_SIZE_BITS; + const PAGE_MASK = PAGE_SIZE - 1; // Wasm32 pointer size is 4 bytes + + const PTR_SIZE_BITS = 2; + const PTR_SIZE = 1 << PTR_SIZE_BITS; + const PTR_MASK = PTR_SIZE - 1; + const PTR_VIEW = Uint32Array; + const BLOCK_OVERHEAD = PTR_SIZE; + + function assert(x) { + if (!x) throw Error("assertion failed"); + return x; + } + + Error.stackTraceLimit = 50; + + function trimStacktrace(stack, levels) { + return stack.split(/\r?\n/).slice(1 + levels); + } + + function isTLSF(stack) { + return stack[0].startsWith(" at ~lib/rt/tlsf/"); + } + + class Rtrace { + constructor(options) { + this.options = options || {}; + + this.onerror = this.options.onerror || function () { + /* nop */ + }; + + this.oninfo = this.options.oninfo || function () { + /* nop */ + }; + + this.memory = null; + this.shadow = null; + this.shadowStart = 0x100000000; + this.refCounts = new Map(); + this.blocks = new Map(); + this.allocSites = new Map(); + this.allocCount = 0; + this.resizeCount = 0; + this.moveCount = 0; + this.freeCount = 0; + this.incrementCount = 0; + this.decrementCount = 0; // The following hooks cannot just be on the prototype but must be + // bound so the Rtrace instance can be used as a WebAssembly import. + + this.onalloc = this.onalloc.bind(this); + this.onresize = this.onresize.bind(this); + this.onmove = this.onmove.bind(this); + this.onfree = this.onfree.bind(this); + this.onincrement = this.onincrement.bind(this); + this.ondecrement = this.ondecrement.bind(this); + this.env = { + load_ptr: this.load_ptr.bind(this), + load_val_i32: this.load_val_i32.bind(this), + load_val_i64: this.load_val_i64.bind(this), + load_val_f32: this.load_val_f32.bind(this), + load_val_f64: this.load_val_f64.bind(this), + store_ptr: this.store_ptr.bind(this), + store_val_i32: this.store_val_i32.bind(this), + store_val_i64: this.store_val_i64.bind(this), + store_val_f32: this.store_val_f32.bind(this), + store_val_f64: this.store_val_f64.bind(this) + }; + } + /** Synchronizes the shadow memory with the module's memory. */ + + + syncShadow() { + if (!this.memory) { + this.memory = assert(this.options.getMemory()); + this.shadow = new WebAssembly.Memory({ + initial: (this.memory.buffer.byteLength + PAGE_MASK & ~PAGE_MASK) >>> PAGE_SIZE_BITS + }); + } else { + var diff = this.memory.buffer.byteLength - this.shadow.buffer.byteLength; + if (diff > 0) this.shadow.grow(diff >>> 16); + } + } + /** Marks a block's presence in shadow memory. */ + + + markShadow(info, oldSize = 0) { + assert(this.shadow && this.shadow.byteLength == this.memory.byteLength); + assert((info.size & PTR_MASK) == 0); + + if (info.ptr < this.shadowStart) { + this.shadowStart = info.ptr; + } + + var len = info.size >>> PTR_SIZE_BITS; + var view = new PTR_VIEW(this.shadow.buffer, info.ptr, len); + var errored = false; + var start = oldSize >>> PTR_SIZE_BITS; + + for (let i = 0; i < start; ++i) { + if (view[i] != info.ptr && !errored) { + this.onerror(Error("shadow region mismatch: " + view[i] + " != " + info.ptr), info); + errored = true; + } + } + + errored = false; + + for (let i = start; i < len; ++i) { + if (view[i] != 0 && !errored) { + this.onerror(Error("shadow region already in use: " + view[i] + " != 0"), info); + errored = true; + } + + view[i] = info.ptr; + } + } + /** Unmarks a block's presence in shadow memory. */ + + + unmarkShadow(info, oldSize = info.size) { + assert(this.shadow && this.shadow.byteLength == this.memory.byteLength); + var len = oldSize >>> PTR_SIZE_BITS; + var view = new PTR_VIEW(this.shadow.buffer, info.ptr, len); + var errored = false; + var start = 0; + + if (oldSize != info.size) { + assert(oldSize > info.size); + start = info.size >>> PTR_SIZE_BITS; + } + + for (let i = 0; i < len; ++i) { + if (view[i] != info.ptr && !errored) { + this.onerror(Error("shadow region mismatch: " + view[i] + " != " + info.ptr), info); + errored = true; + } + + if (i >= start) view[i] = 0; + } + } + /** Performs an access to shadow memory. */ + + + accessShadow(ptr, size, isLoad) { + this.syncShadow(); + if (ptr < this.shadowStart) return; + var value = new Uint32Array(this.shadow.buffer, ptr & ~PTR_MASK, 1)[0]; + if (value != 0) return; // FIXME: this is extremely slow + + let stack = trimStacktrace(new Error().stack, 2); + + if (!isTLSF(stack)) { + this.onerror(new Error("OOB " + (isLoad ? "load" : "store") + 8 * size + " at address " + ptr + "\n" + stack.join("\n"))); + } + } + /** Obtains information about a block. */ + + + getBlockInfo(ptr) { + var header = new Uint32Array(this.memory.buffer, ptr, 5); + var mmInfo = header[0]; + var gcInfo = header[1]; + var gcInfo2 = header[2]; + const mmTags = [[], ["FREE"], ["LEFTFREE"], ["FREE", "LEFTFREE"]]; // 2=LEFTFREE, 1=FREE + + const gcColor = ["BLACK", "GRAY", "WHITE", "PURPLE"]; + var size = mmInfo & ~3; + return { + ptr, + size: BLOCK_OVERHEAD + size, + // block size + header: { + mmInfo: { + tags: mmTags[mmInfo & 3], + size: size + }, + gcInfo: { + buffered: gcInfo >>> 31 === 1, + color: gcColor[gcInfo << 1 >>> 29], + rc: gcInfo << 4 >>> 4 + }, + gcInfo2, + rtId: header[3], + rtSize: header[4] + } + }; + } + /** Checks if rtrace is active, i.e. at least one event has occurred. */ + + + get active() { + return Boolean(this.allocCount || this.moveCount || this.freeCount || this.incrementCount || this.decrementCount); + } + /** Checks if there are any leaks and emits them via `oninfo`. Returns the number of live blocks. */ + + + check() { + if (this.refCounts.size == 1) return 0; // purerc roots + + if (this.oninfo) { + for (let [ptr, rc] of this.refCounts) { + this.oninfo("LEAKING " + ptr + " @ " + rc); + } + } + + return this.refCounts.size; + } // Runtime instrumentation + + + onalloc(ptr) { + this.syncShadow(); + ++this.allocCount; + var info = this.getBlockInfo(ptr); + + if (this.refCounts.has(ptr)) { + this.onerror(Error("duplicate alloc: " + ptr), info); + } else { + this.oninfo("ALLOC " + ptr + ".." + (ptr + info.size)); + this.markShadow(info); + this.refCounts.set(ptr, 0); + this.blocks.set(ptr, Object.assign(info, { + allocStack: trimStacktrace(new Error().stack, 1) // strip onalloc + + })); + } + } + + onresize(ptr, oldSize) { + this.syncShadow(); + ++this.resizeCount; + var info = this.getBlockInfo(ptr); + + if (!this.refCounts.has(ptr)) { + this.onerror(Error("orphaned resize: " + ptr), info); + } else { + this.oninfo("RESIZE " + ptr + ".." + (ptr + info.size) + " (" + oldSize + "->" + info.size + ")"); + + if (info.size > oldSize) { + this.markShadow(info, BLOCK_OVERHEAD + oldSize); + } else if (info.size < oldSize) { + this.unmarkShadow(info, BLOCK_OVERHEAD + oldSize); + } + } + } + + onmove(oldPtr, newPtr) { + this.syncShadow(); + ++this.moveCount; + var oldInfo = this.getBlockInfo(oldPtr); + var newInfo = this.getBlockInfo(newPtr); + + if (!this.refCounts.has(oldPtr)) { + this.onerror(Error("orphaned move (old): " + oldPtr), oldInfo); + } else { + if (!this.refCounts.has(newPtr)) { + this.onerror(Error("orphaned move (new): " + newPtr), newInfo); + } else { + let newRc = this.refCounts.get(newPtr); + + if (newRc != 0) { + this.onerror(Error("invalid realloc: " + oldPtr + " -> " + newPtr + " @ " + newRc), oldInfo); + } else { + let oldRc = this.refCounts.get(oldPtr); + this.oninfo("MOVE " + oldPtr + ".." + (oldPtr + oldInfo.size) + " @ " + oldRc + " -> " + newPtr + ".." + (newPtr + newInfo.size)); + this.refCounts.set(newPtr, oldRc); // calls new alloc before and old free after + } + } + } + } + + onfree(ptr) { + this.syncShadow(); + ++this.freeCount; + var info = this.getBlockInfo(ptr); + + if (!this.refCounts.has(ptr)) { + this.onerror(Error("orphaned free: " + ptr), info); + } else { + this.oninfo("FREE " + ptr + ".." + (ptr + info.size) + " @ " + this.refCounts.get(ptr)); + this.refCounts.delete(ptr); + this.unmarkShadow(info); + let block = this.blocks.get(ptr); + block.freeStack = trimStacktrace(new Error().stack, 1); // strip onfree + } + } + + onincrement(ptr) { + this.syncShadow(); + ++this.incrementCount; + var info = this.getBlockInfo(ptr); + + if (!this.refCounts.has(ptr)) { + this.onerror(Error("orphaned increment: " + ptr), info); + } else { + let rc = this.refCounts.get(ptr); + this.oninfo("++ " + ptr + " @ " + rc + "->" + (rc + 1)); + this.refCounts.set(ptr, rc + 1); + } + } + + ondecrement(ptr) { + this.syncShadow(); + ++this.decrementCount; + var info = this.getBlockInfo(ptr); + + if (!this.refCounts.has(ptr)) { + this.onerror(Error("orphaned decrement: " + ptr), info); + } else { + let rc = this.refCounts.get(ptr); + + if (rc < 1) { + this.onerror(Error("invalid decrement: " + ptr + " @ " + rc), info); + } else { + this.oninfo("-- " + ptr + " @ " + rc + "->" + (rc - 1)); + this.refCounts.set(ptr, rc - 1); + } + } + } // Memory instrumentation + + + load_ptr(id, bytes, offset, address) { + this.accessShadow(address + offset, bytes, true); + return address; + } + + load_val_i32(id, value) { + return value; + } + + load_val_i64(id, value) { + return value; + } + + load_val_f32(id, value) { + return value; + } + + load_val_f64(id, value) { + return value; + } + + store_ptr(id, bytes, offset, address) { + this.accessShadow(address + offset, bytes, false); + return address; + } + + store_val_i32(id, value) { + return value; + } + + store_val_i64(id, value) { + return value; + } + + store_val_f32(id, value) { + return value; + } + + store_val_f64(id, value) { + return value; + } + + } + + exports.Rtrace = Rtrace; + var _default = { + Rtrace + }; + exports.default = _default; + return exports; +})({}); +if (typeof define === 'function' && define.amd) define([], function() { return rtrace; }); +else if (typeof module === 'object' && typeof exports==='object') module.exports = rtrace; diff --git a/lib/rtrace/umd/package.json b/lib/rtrace/umd/package.json new file mode 100644 index 0000000000..31f6a4909a --- /dev/null +++ b/lib/rtrace/umd/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "commonjs" +} \ No newline at end of file diff --git a/package.json b/package.json index 839baff6a2..c87232056b 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,10 @@ "./lib/loader": { "import": "./lib/loader/index.js", "require": "./lib/loader/umd/index.js" + }, + "./lib/rtrace": { + "import": "./lib/rtrace/index.js", + "require": "./lib/rtrace/umd/index.js" } }, "bin": { @@ -85,10 +89,6 @@ "astest": "ts-node tests/bootstrap" }, "releaseFiles": [ - "lib/rtrace/index.d.ts", - "lib/rtrace/index.js", - "lib/rtrace/README.md", - "lib/rtrace/package.json", "lib/loader/index.d.ts", "lib/loader/index.js", "lib/loader/package.json", @@ -96,6 +96,13 @@ "lib/loader/umd/index.js", "lib/loader/umd/package.json", "lib/loader/README.md", + "lib/rtrace/index.d.ts", + "lib/rtrace/index.js", + "lib/rtrace/package.json", + "lib/rtrace/umd/index.d.ts", + "lib/rtrace/umd/index.js", + "lib/rtrace/umd/package.json", + "lib/rtrace/README.md", "bin/", "cli/", "dist/", diff --git a/tests/bootstrap/index.ts b/tests/bootstrap/index.ts index d31c57cece..1f66ea7c22 100644 --- a/tests/bootstrap/index.ts +++ b/tests/bootstrap/index.ts @@ -2,8 +2,8 @@ import * as fs from "fs"; import * as path from "path"; import * as v8 from "v8"; import * as binaryen from "binaryen"; -import * as loader from "../../lib/loader/umd"; -import { Rtrace } from "../../lib/rtrace"; +import { instantiate } from "../../lib/loader/umd"; +import { Rtrace } from "../../lib/rtrace/umd"; import * as find from "../../cli/util/find"; import AssemblyScript from "../../out/assemblyscript"; @@ -31,7 +31,7 @@ async function test(build: string): Promise { } }); - const { exports: asc } = await loader.instantiate( + const { exports: asc } = await instantiate( fs.promises.readFile(`${ __dirname }/../../out/assemblyscript.${ build }.wasm`), { binaryen, diff --git a/tests/compiler.js b/tests/compiler.js index 06692621e1..0ea954aefb 100644 --- a/tests/compiler.js +++ b/tests/compiler.js @@ -8,7 +8,7 @@ const colorsUtil = require("../cli/util/colors"); const optionsUtil = require("../cli/util/options"); const diff = require("./util/diff"); const asc = require("../cli/asc.js"); -const { Rtrace } = require("../lib/rtrace"); +const { Rtrace } = require("../lib/rtrace/umd"); const cluster = require("cluster"); const coreCount = require("physical-cpu-count"); From e685f2a051c66595b5adb5d3b9f2f7264dd44d58 Mon Sep 17 00:00:00 2001 From: dcode Date: Thu, 22 Oct 2020 17:28:55 +0200 Subject: [PATCH 2/5] consistent naming --- lib/rtrace/README.md | 2 +- lib/rtrace/index.d.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/rtrace/README.md b/lib/rtrace/README.md index a6b0f18622..13e29c38a0 100644 --- a/lib/rtrace/README.md +++ b/lib/rtrace/README.md @@ -1,4 +1,4 @@ -# RTrace +# Rtrace A tiny utility that records allocations, retains, releases and frees performed by the runtime and emits an error if something is off. Also checks for leaks. diff --git a/lib/rtrace/index.d.ts b/lib/rtrace/index.d.ts index 59b2e54a95..a5703cb4f9 100644 --- a/lib/rtrace/index.d.ts +++ b/lib/rtrace/index.d.ts @@ -30,7 +30,7 @@ export declare interface RtraceOptions { export declare class Rtrace { [key: string]: unknown; // can be used as a Wasm import - /** Creates a new `RTrace` instance. */ + /** Creates a new `Rtrace` instance. */ constructor(options: RtraceOptions); /** Checks if rtrace is active, i.e. at least one event has occurred. */ From 4767fc2b31ae835600b043c6fca2d92462920ed0 Mon Sep 17 00:00:00 2001 From: dcode Date: Thu, 22 Oct 2020 17:30:36 +0200 Subject: [PATCH 3/5] consistent README --- lib/rtrace/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rtrace/README.md b/lib/rtrace/README.md index 13e29c38a0..1a560f185e 100644 --- a/lib/rtrace/README.md +++ b/lib/rtrace/README.md @@ -1,4 +1,4 @@ -# Rtrace +# AssemblyScript Rtrace A tiny utility that records allocations, retains, releases and frees performed by the runtime and emits an error if something is off. Also checks for leaks. From 7080cd2b8079679edc71df84a1d43cdee99ddf4a Mon Sep 17 00:00:00 2001 From: Daniel Wirtz Date: Thu, 22 Oct 2020 19:39:09 +0200 Subject: [PATCH 4/5] Update lib/rtrace/umd/index.js Co-authored-by: Max Graey --- lib/rtrace/umd/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/rtrace/umd/index.js b/lib/rtrace/umd/index.js index 5bdbaaa7e0..c6ee43acf7 100644 --- a/lib/rtrace/umd/index.js +++ b/lib/rtrace/umd/index.js @@ -28,8 +28,10 @@ var rtrace = (function(exports) { return stack.split(/\r?\n/).slice(1 + levels); } + const RT_TLSF = "~lib/rt/tlsf/"; + // ... function isTLSF(stack) { - return stack[0].startsWith(" at ~lib/rt/tlsf/"); + return stack[0].startsWith(` at ${RT_TLSF}`); } class Rtrace { From 47727bfa614768d3f93e3479e5a254f5e9097805 Mon Sep 17 00:00:00 2001 From: dcode Date: Thu, 22 Oct 2020 20:17:17 +0200 Subject: [PATCH 5/5] fix --- lib/rtrace/index.js | 4 +++- lib/rtrace/umd/index.js | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/rtrace/index.js b/lib/rtrace/index.js index f28dc7fbfc..7f1a17111b 100644 --- a/lib/rtrace/index.js +++ b/lib/rtrace/index.js @@ -11,6 +11,8 @@ const PTR_VIEW = Uint32Array; const BLOCK_OVERHEAD = PTR_SIZE; +const RT_TLSF = "~lib/rt/tlsf/"; + function assert(x) { if (!x) throw Error("assertion failed"); return x; @@ -23,7 +25,7 @@ function trimStacktrace(stack, levels) { } function isTLSF(stack) { - return stack[0].startsWith(" at ~lib/rt/tlsf/"); + return stack[0].startsWith(` at ${RT_TLSF}`); } export class Rtrace { diff --git a/lib/rtrace/umd/index.js b/lib/rtrace/umd/index.js index c6ee43acf7..7c05e4bd9e 100644 --- a/lib/rtrace/umd/index.js +++ b/lib/rtrace/umd/index.js @@ -16,6 +16,7 @@ var rtrace = (function(exports) { const PTR_MASK = PTR_SIZE - 1; const PTR_VIEW = Uint32Array; const BLOCK_OVERHEAD = PTR_SIZE; + const RT_TLSF = "~lib/rt/tlsf/"; function assert(x) { if (!x) throw Error("assertion failed"); @@ -28,8 +29,6 @@ var rtrace = (function(exports) { return stack.split(/\r?\n/).slice(1 + levels); } - const RT_TLSF = "~lib/rt/tlsf/"; - // ... function isTLSF(stack) { return stack[0].startsWith(` at ${RT_TLSF}`); }