From 72cf949e05c2b7013efca6d88571e3abd84f08dc Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 13 Oct 2020 23:29:47 -0700 Subject: [PATCH 01/13] feat: typedef generation & type checking --- .eslintrc | 3 + .github/workflows/typecheck.yml | 27 +++++ package.json | 25 ++++- src/decision-engine/index.js | 119 +++++++++++++++++--- src/decision-engine/interface.ts | 51 +++++++++ src/decision-engine/ledger.js | 35 ++++++ src/decision-engine/req-queue.js | 134 ++++++++++++++++++----- src/decision-engine/task-merger.js | 49 ++------- src/index.js | 130 ++++++++++++++++------ src/network.js | 101 +++++++++++++---- src/notifications.js | 31 ++++-- src/stats/index.js | 40 +++++-- src/stats/stat.js | 60 ++++++++-- src/types.ts | 64 +++++++++++ src/types/message/entry.js | 12 ++ src/types/message/index.js | 70 +++++++++++- src/types/wantlist/entry.js | 14 +++ src/types/wantlist/index.js | 34 ++++++ src/utils/index.js | 63 ++++++++--- src/utils/sorted-map.js | 68 +++++++++++- src/want-manager/index.js | 14 ++- src/want-manager/msg-queue.js | 13 ++- test/network/gen-bitswap-network.node.js | 7 +- test/utils/mocks.js | 4 +- tsconfig.json | 42 +++++++ 25 files changed, 1011 insertions(+), 199 deletions(-) create mode 100644 .eslintrc create mode 100644 .github/workflows/typecheck.yml create mode 100644 src/decision-engine/interface.ts create mode 100644 src/types.ts create mode 100644 tsconfig.json diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 00000000..ea565dd6 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,3 @@ +{ + "extends": "ipfs" +} diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml new file mode 100644 index 00000000..624310c2 --- /dev/null +++ b/.github/workflows/typecheck.yml @@ -0,0 +1,27 @@ +on: + push: + branches: + - master + - main + - default + pull_request: + branches: + - '**' + +name: Typecheck +jobs: + check: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [12.x] + steps: + - uses: actions/checkout@v1 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Install dependencies + run: npm install + - name: Typecheck + uses: gozala/typescript-error-reporter-action@v1.0.4 diff --git a/package.json b/package.json index 9f03298b..11e2ee0c 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,13 @@ "./test/utils/create-libp2p-node": false, "./test/utils/create-temp-repo-nodejs.js": "./test/utils/create-temp-repo-browser.js" }, + "typesVersions": { + "*": { + "*": [ + "dist/*" + ] + } + }, "files": [ "dist", "src" @@ -17,14 +24,18 @@ "test:browser": "aegir test -t browser -t webworker", "test:node": "aegir test -t node", "lint": "aegir lint", + "check": "tsc --noEmit", "release": "aegir release", "release-minor": "aegir release --type minor", "release-major": "aegir release --type major", "bench": "node benchmarks/index", - "build": "aegir build", + "build": "npm run build:js & npm run build:types", + "build:js": "aegir build", + "build:types": "tsc --emitDeclarationOnly --declarationDir dist", "coverage": "aegir coverage --provider codecov", "docs": "aegir docs", - "benchmarks": "node test/benchmarks/get-many" + "benchmarks": "node test/benchmarks/get-many", + "prepare": "npm run build:types" }, "repository": { "type": "git", @@ -43,7 +54,7 @@ "homepage": "https://github.com/ipfs/js-ipfs-bitswap#readme", "devDependencies": { "@nodeutils/defaults-deep": "^1.1.0", - "aegir": "^26.0.0", + "aegir": "^27.0.0", "benchmark": "^2.1.4", "delay": "^4.3.0", "ipfs-repo": "^6.0.1", @@ -69,15 +80,17 @@ "rimraf": "^3.0.0", "sinon": "^9.0.0", "stats-lite": "^2.2.0", - "uuid": "^8.0.0" + "uuid": "^8.0.0", + "typescript": "^4.0.3", + "@types/debug": "^4.1.5" }, "dependencies": { "abort-controller": "^3.0.0", "any-signal": "^1.1.0", "bignumber.js": "^9.0.0", "cids": "^1.0.0", - "debug": "^4.1.0", - "ipld-block": "^0.10.0", + "debug": "^4.2.0", + "ipld-block": "git://github.com/ipld/js-ipld-block#typegen", "it-length-prefixed": "^3.0.0", "it-pipe": "^1.1.0", "just-debounce-it": "^1.1.0", diff --git a/src/decision-engine/index.js b/src/decision-engine/index.js index b873e9dd..a7b7f7e8 100644 --- a/src/decision-engine/index.js +++ b/src/decision-engine/index.js @@ -26,6 +26,16 @@ const TARGET_MESSAGE_SIZE = 16 * 1024 const MAX_SIZE_REPLACE_HAS_WITH_BLOCK = 1024 class DecisionEngine { + /** + * + * @param {PeerId} peerId + * @param {*} blockstore + * @param {import('../network')} network + * @param {Stats} stats + * @param {Object} [opts] + * @param {number} [opts.targetMessageSize] + * @param {number} [opts.maxSizeReplaceHasWithBlock] + */ constructor (peerId, blockstore, network, stats, opts) { this._log = logger(peerId, 'engine') this.blockstore = blockstore @@ -34,6 +44,7 @@ class DecisionEngine { this._opts = this._processOpts(opts) // A list of of ledgers by their partner id + /** @type {Map} */ this.ledgerMap = new Map() this._running = false @@ -112,7 +123,7 @@ class DecisionEngine { // If there's nothing in the message, bail out if (msg.empty) { - this._requestQueue.tasksDone(peerId, tasks) + peerId && this._requestQueue.tasksDone(peerId, tasks) // Trigger the next round of task processing this._scheduleProcessTasks() @@ -122,32 +133,36 @@ class DecisionEngine { try { // Send the message - await this.network.sendMessage(peerId, msg) + peerId && await this.network.sendMessage(peerId, msg) // Peform sent message accounting for (const block of blocks.values()) { - this.messageSent(peerId, block) + peerId && this.messageSent(peerId, block) } } catch (err) { this._log.error(err) } // Free the tasks up from the request queue - this._requestQueue.tasksDone(peerId, tasks) + peerId && this._requestQueue.tasksDone(peerId, tasks) // Trigger the next round of task processing this._scheduleProcessTasks() } + /** + * @param {PeerId} peerId + * @returns {Map} + */ wantlistForPeer (peerId) { const peerIdStr = peerId.toB58String() - if (!this.ledgerMap.has(peerIdStr)) { - return new Map() - } - - return this.ledgerMap.get(peerIdStr).wantlist.sortedEntries() + const ledger = this.ledgerMap.get(peerIdStr) + return ledger ? ledger.wantlist.sortedEntries() : new Map() } + /** + * @param {PeerId} peerId + */ ledgerForPeer (peerId) { const peerIdStr = peerId.toB58String() @@ -164,12 +179,20 @@ class DecisionEngine { } } + /** + * @returns {PeerId[]} + */ peers () { return Array.from(this.ledgerMap.values()).map((l) => l.partner) } - // Receive blocks either from an incoming message from the network, or from - // blocks being added by the client on the localhost (eg IPFS add) + /** + * Receive blocks either from an incoming message from the network, or from + * blocks being added by the client on the localhost (eg IPFS add) + * + * @param {Block[]} blocks + * @returns {void} + */ receivedBlocks (blocks) { if (!blocks.length) { return @@ -211,7 +234,13 @@ class DecisionEngine { this._scheduleProcessTasks() } - // Handle incoming messages + /** + * Handle incoming messages + * + * @param {PeerId} peerId + * @param {Message} msg + * @returns {Promise} + */ async messageReceived (peerId, msg) { const ledger = this._findOrCreate(peerId) @@ -251,12 +280,24 @@ class DecisionEngine { this._scheduleProcessTasks() } + /** + * @private + * @param {PeerId} peerId + * @param {CID[]} cids + * @returns {void} + */ _cancelWants (peerId, cids) { for (const c of cids) { this._requestQueue.remove(c.toString(), peerId) } } + /** + * @private + * @param {PeerId} peerId + * @param {BitswapMessageEntry[]} wants + * @returns {Promise} + */ async _addWants (peerId, wants) { // Get the size of each wanted block const blockSizes = await this._getBlockSizes(wants.map(w => w.cid)) @@ -320,11 +361,21 @@ class DecisionEngine { blockSize <= this._opts.maxSizeReplaceHasWithBlock } + /** + * @private + * @param {CID[]} cids + * @returns {Promise>} + */ async _getBlockSizes (cids) { const blocks = await this._getBlocks(cids) return new Map([...blocks].map(([k, v]) => [k, v.data.length])) } + /** + * @private + * @param {CID[]} cids + * @returns {Promise>} + */ async _getBlocks (cids) { const res = new Map() await Promise.all(cids.map(async (cid) => { @@ -347,7 +398,14 @@ class DecisionEngine { }) } - // Clear up all accounting things after message was sent + /** + * Clear up all accounting things after message was sent + * + * @param {PeerId} peerId + * @param {Object} [block] + * @param {Uint8Array} block.data + * @param {CID} [block.cid] + */ messageSent (peerId, block) { const ledger = this._findOrCreate(peerId) ledger.sentBytes(block ? block.data.length : 0) @@ -356,15 +414,29 @@ class DecisionEngine { } } + /** + * @param {PeerId} peerId + * @returns {number} + */ numBytesSentTo (peerId) { return this._findOrCreate(peerId).accounting.bytesSent } + /** + * @param {PeerId} peerId + * @returns {number} + */ + numBytesReceivedFrom (peerId) { return this._findOrCreate(peerId).accounting.bytesRecv } - peerDisconnected (peerId) { + /** + * + * @param {PeerId} _peerId + * @returns {void} + */ + peerDisconnected (_peerId) { // if (this.ledgerMap.has(peerId.toB58String())) { // this.ledgerMap.delete(peerId.toB58String()) // } @@ -373,10 +445,16 @@ class DecisionEngine { // in the peer request queue } + /** + * @private + * @param {PeerId} peerId + * @returns {Ledger} + */ _findOrCreate (peerId) { const peerIdStr = peerId.toB58String() - if (this.ledgerMap.has(peerIdStr)) { - return this.ledgerMap.get(peerIdStr) + const ledger = this.ledgerMap.get(peerIdStr) + if (ledger) { + return ledger } const l = new Ledger(peerId) @@ -399,3 +477,12 @@ class DecisionEngine { } module.exports = DecisionEngine + +/** + * @typedef {import('../types').PeerId} PeerId + * @typedef {import('../stats')} Stats + * @typedef {import('../types').BlockData} BlockData + * @typedef {import('../types').Block} Block + * @typedef {import('../types/message/entry')} BitswapMessageEntry + * @typedef {import('../types/wantlist/entry')} WantListEntry + */ diff --git a/src/decision-engine/interface.ts b/src/decision-engine/interface.ts new file mode 100644 index 00000000..d318d1df --- /dev/null +++ b/src/decision-engine/interface.ts @@ -0,0 +1,51 @@ +export interface TaskMerger { + /** + * Ggiven the existing tasks with the same topic, does the task add some new + * information? Used to decide whether to merge the task or ignore it. + */ + hasNewInfo (task:Task, tasksWithTopic:Task[]): boolean + + /** + * Merge the information from the task into the existing pending task. + */ + merge (newTask, existingTask): void +} + +export interface Task { + /** + * A name for the Task (like an id but not necessarily unique) + */ + topic: string + /** + * Priority for the Task (tasks are ordered by priority per peer). + */ + priority: number + /** + * The size of the task, e.g. the number of bytes in a block. + */ + size: number + + data: TaskData +} + +export interface TaskData { + /** + * The size of the block, if known (if we don't have the block this is zero) + */ + blockSize: number + /** + * Indicates if the request is for a block or for a HAVE. + */ + isWantBlock: boolean + /** + * Indicates if we have the block. + */ + haveBlock: boolean + /** + * Indicates whether to send a DONT_HAVE response if we don't have the block. + * If this is `false` and we don't have the block, we just ignore the + * want-block request (useful for discovery where we query lots of peers but + * don't want a response unless the peer has the block). + */ + sendDontHave: boolean +} diff --git a/src/decision-engine/ledger.js b/src/decision-engine/ledger.js index 407f6f2c..0ef7ea0a 100644 --- a/src/decision-engine/ledger.js +++ b/src/decision-engine/ledger.js @@ -3,6 +3,9 @@ const Wantlist = require('../types/wantlist') class Ledger { + /** + * @param {PeerId} peerId + */ constructor (peerId) { this.partner = peerId this.wantlist = new Wantlist() @@ -16,33 +19,65 @@ class Ledger { } } + /** + * @param {number} n + */ sentBytes (n) { this.exchangeCount++ this.lastExchange = (new Date()).getTime() this.accounting.bytesSent += n } + /** + * @param {number} n + */ receivedBytes (n) { this.exchangeCount++ this.lastExchange = (new Date()).getTime() this.accounting.bytesRecv += n } + /** + * + * @param {CID} cid + * @param {number} priority + * @param {WantType} [wantType] + * @returns {void} + */ wants (cid, priority, wantType) { this.wantlist.add(cid, priority, wantType) } + /** + * @param {CID} cid + * @returns {void} + */ + cancelWant (cid) { this.wantlist.remove(cid) } + /** + * @param {CID} cid + * @returns {WantListEntry|void} + */ wantlistContains (cid) { return this.wantlist.contains(cid) } + /** + * @returns {number} + */ debtRatio () { return (this.accounting.bytesSent / (this.accounting.bytesRecv + 1)) // +1 is to prevent division by zero } } module.exports = Ledger + +/** + * @typedef {import('../types').PeerId} PeerId + * @typedef {import('../types').CID} CID + * @typedef {import('../types').WantType} WantType + * @typedef {import('../types/wantlist/entry')} WantListEntry + */ diff --git a/src/decision-engine/req-queue.js b/src/decision-engine/req-queue.js index c8e0f937..f3b7512c 100644 --- a/src/decision-engine/req-queue.js +++ b/src/decision-engine/req-queue.js @@ -2,26 +2,12 @@ const SortedMap = require('../utils/sorted-map') -/** - * @typedef {Object} Task - * @property {string} topic - a name for the Task (like an id but not necessarily unique) - * @property {number} priority - tasks are ordered by priority per peer - * @property {number} size - the size of the task, eg the number of bytes in a block - */ - -/** - * @typedef {Object} TaskMerger - * @property {function(task, tasksWithTopic)} hasNewInfo - given the existing - * tasks with the same topic, does the task add some new information? - * Used to decide whether to merge the task or ignore it. - * @property {function(task, existingTask)} merge - merge the information from - * the given task into the existing task (with the same topic) - */ - /** * The task merger that is used by default. * Assumes that new tasks do not add any information over existing tasks, * and doesn't try to merge. + * + * @type {TaskMerger} */ const DefaultTaskMerger = { hasNewInfo () { @@ -44,13 +30,16 @@ class RequestQueue { */ constructor (taskMerger) { this._taskMerger = taskMerger || DefaultTaskMerger - this._byPeer = new SortedMap([], PeerTasks.compare, true) + /** @type {SortedMap} */ + this._byPeer = new SortedMap([], PeerTasks.compare) } /** * Push tasks onto the queue for the given peer + * * @param {PeerId} peerId - * @param {Task} tasks + * @param {Task[]} tasks + * @returns {void} */ pushTasks (peerId, tasks) { let peerTasks = this._byPeer.get(peerId.toB58String()) @@ -69,17 +58,18 @@ class RequestQueue { * the total size is at least targetMinBytes. * This puts the popped tasks into the "active" state, meaning they are * actively being processed (and cannot be modified). + * * @param {number} targetMinBytes - the minimum total size of tasks to pop - * @returns {Object} + * @returns {PopTaskResult} */ popTasks (targetMinBytes) { - if (this._byPeer.size === 0) { - return { tasks: [], pendingSize: 0 } - } - // Get the queue of tasks for the best peer and pop off tasks up to // targetMinBytes const peerTasks = this._head() + if (peerTasks === undefined) { + return { tasks: [], pendingSize: 0 } + } + const { tasks, pendingSize } = peerTasks.popTasks(targetMinBytes) if (tasks.length === 0) { return { tasks, pendingSize } @@ -100,7 +90,16 @@ class RequestQueue { } } + /** + * @private + * @returns {PeerTasks|undefined} + */ _head () { + // Shortcut + if (this._byPeer.size === 0) { + return undefined + } + for (const [, v] of this._byPeer) { return v } @@ -109,8 +108,10 @@ class RequestQueue { /** * Remove the task with the given topic for the given peer. + * * @param {string} topic * @param {PeerId} peerId + * @returns {void} */ remove (topic, peerId) { const peerTasks = this._byPeer.get(peerId.toB58String()) @@ -119,8 +120,10 @@ class RequestQueue { /** * Called when the tasks for the given peer complete. + * * @param {PeerId} peerId * @param {Task[]} tasks + * @returns {void} */ tasksDone (peerId, tasks) { const peerTasks = this._byPeer.get(peerId.toB58String()) @@ -158,7 +161,9 @@ class PeerTasks { /** * Push tasks onto the queue. + * * @param {Task[]} tasks + * @returns {void} */ pushTasks (tasks) { for (const t of tasks) { @@ -166,6 +171,12 @@ class PeerTasks { } } + /** + * @private + * @param {Task} task + * @returns {void} + */ + _pushTask (task) { // If the new task doesn't add any more information over what we // already have in the active queue, then we can skip the new task @@ -194,8 +205,14 @@ class PeerTasks { this._pending.add(task) } - // Indicates whether the new task adds any more information over tasks that are - // already in the active task queue + /** + * Indicates whether the new task adds any more information over tasks that are + * already in the active task queue + * + * @private + * @param {Task} task + * @returns {boolean} + */ _taskHasMoreInfoThanActiveTasks (task) { const tasksWithTopic = [] for (const activeTask of this._active) { @@ -214,6 +231,7 @@ class PeerTasks { /** * Pop tasks off the queue such that the total size is at least targetMinBytes + * * @param {number} targetMinBytes * @returns {Object} */ @@ -243,7 +261,9 @@ class PeerTasks { /** * Called when a task completes. * Note: must be the same reference as returned from popTasks. + * * @param {Task} task + * @returns {void} */ taskDone (task) { if (this._active.has(task)) { @@ -254,7 +274,9 @@ class PeerTasks { /** * Remove pending tasks with the given topic + * * @param {string} topic + * @returns {void} */ remove (topic) { this._pending.delete(topic) @@ -262,13 +284,21 @@ class PeerTasks { /** * No work to be done, this PeerTasks object can be freed. + * * @returns {boolean} */ isIdle () { - return this._pending.length === 0 && this._active.length === 0 + return this._pending.length === 0 && this._active.size === 0 } - // Compare PeerTasks + /** + * Compare PeerTasks + * + * @template Key + * @param {[Key, PeerTasks]} a + * @param {[Key, PeerTasks]} b + * @returns {number} + */ static compare (a, b) { // Move peers with no pending tasks to the back of the queue if (a[1]._pending.length === 0) { @@ -294,6 +324,7 @@ class PeerTasks { */ class PendingTasks { constructor () { + /** @type {SortedMap} */ this._tasks = new SortedMap([], this._compare) } @@ -301,15 +332,26 @@ class PendingTasks { return this._tasks.size } - // Sum of the size of all pending tasks + /** + * Sum of the size of all pending tasks + * + * @type {number} + **/ get totalSize () { return [...this._tasks.values()].reduce((a, t) => a + t.task.size, 0) } + /** + * @param {string} topic + * @returns {Task|void} + */ get (topic) { return (this._tasks.get(topic) || {}).task } + /** + * @param {Task} task + */ add (task) { this._tasks.set(task.topic, { created: Date.now(), @@ -317,6 +359,10 @@ class PendingTasks { }) } + /** + * @param {string} topic + * @returns {void} + */ delete (topic) { this._tasks.delete(topic) } @@ -326,7 +372,13 @@ class PendingTasks { return [...this._tasks.values()].map(i => i.task) } - // Update the priority of the task with the given topic, and update the order + /** + * Update the priority of the task with the given topic, and update the order + * + * @param {string} topic + * @param {number} priority + * @returns {void} + **/ updatePriority (topic, priority) { const obj = this._tasks.get(topic) if (!obj) { @@ -338,7 +390,14 @@ class PendingTasks { this._tasks.update(i) } - // Sort by priority desc then FIFO + /** + * Sort by priority desc then FIFO + * + * @param {[string, PendingTask]} a + * @param {[string, PendingTask]} b + * @returns {number} + * @private + */ _compare (a, b) { if (a[1].task.priority === b[1].task.priority) { // FIFO @@ -350,3 +409,18 @@ class PendingTasks { } module.exports = RequestQueue + +/** + * @typedef {Object} PopTaskResult + * @property {PeerId} [peerId] + * @property {Task[]} tasks + * @property {number} pendingSize + * + * @typedef {Object} PendingTask + * @property {number} created + * @property {Task} task + * + * @typedef {import('../types').PeerId} PeerId + * @typedef {import('./interface').Task} Task + * @typedef {import('./interface').TaskMerger} TaskMerger + */ diff --git a/src/decision-engine/task-merger.js b/src/decision-engine/task-merger.js index 293af158..2f16aacc 100644 --- a/src/decision-engine/task-merger.js +++ b/src/decision-engine/task-merger.js @@ -1,9 +1,11 @@ 'use strict' +/** @type {TaskMergerAPI} */ const TaskMerger = { /** * Indicates whether the given task has newer information than the active - * tasks with the same topic + * tasks with the same topic. + * * @param {Task} task * @param {Task[]} tasksWithTopic * @returns {boolean} @@ -38,46 +40,13 @@ const TaskMerger = { }, /** - * Merge the information from the task into the existing pending task + * Merge the information from the given task into the existing task (with the + * same topic) + * * @param {Task} newTask * @param {Task} existingTask */ merge (newTask, existingTask) { - // Tasks look like this: - // { - // topic: "some topic", - // priority: 5, - // - // # The size of the response on the wire. This is used to calculate - // # how many tasks to pop off the request queue and add to a message. - // # If the response is - // # - a HAVE or DONT_HAVE - // # it is the size of the CID + type (HAVE/DONT_HAVE) - // # - a block - // # it is the size of the block - // size: 32, - // - // data: { - // - // # The size of the block, if known (if we don't have the block this is zero) - // blockSize: 128 * 1024, - // - // # Indicates if the request is for a block or for a HAVE - // isWantBlock: false, - // - // # Do we have the block? - // # Note: a block can have size zero. - // haveBlock: true, - // - // # Indicates whether to send a DONT_HAVE response if we don't have - // # the block. - // # If this is false and we don't have the block, we just ignore the - // # want-block request (useful for discovery where we query lots of - // # peers but don't want a response unless the peer has the block). - // sendDontHave: false - // } - // } - // // The merge function ignores the topic and priority as these don't change. // // We may receive new information about a want before the want has been @@ -95,8 +64,6 @@ const TaskMerger = { // 3. Local node receives block for CID1 from peer // In this case we should replace DONT_HAVE with the want, including // updating the task size and block size. - // - const taskData = newTask.data const existingData = existingTask.data @@ -129,3 +96,7 @@ const TaskMerger = { } module.exports = TaskMerger +/** + * @typedef {import('./interface').Task} Task + * @typedef {import('./interface').TaskMerger} TaskMergerAPI + */ diff --git a/src/index.js b/src/index.js index 5c39c00a..d27869a2 100644 --- a/src/index.js +++ b/src/index.js @@ -6,7 +6,7 @@ const DecisionEngine = require('./decision-engine') const Notifications = require('./notifications') const logger = require('./utils').logger const Stats = require('./stats') -const AbortController = require('abort-controller') +const AbortController = require('abort-controller').default const anySignal = require('any-signal') const defaultOptions = { @@ -29,12 +29,16 @@ const statsKeys = [ /** * JavaScript implementation of the Bitswap 'data exchange' protocol * used by IPFS. - * - * @param {Libp2p} libp2p - * @param {Blockstore} blockstore - * @param {Object} options */ class Bitswap { + /** + * @param {LibP2P} libp2p + * @param {BlockStore} blockstore + * @param {Object} options + * @param {boolean} [options.statsEnabled=false] + * @param {number} [options.statsComputeThrottleTimeout=1000] + * @param {number} [options.statsComputeThrottleMaxQueueSize=1000] + */ constructor (libp2p, blockstore, options) { this._libp2p = libp2p this._log = logger(this.peerId) @@ -62,11 +66,20 @@ class Bitswap { this.notifications = new Notifications(this.peerId) } + /** + * @type {PeerId} + */ get peerId () { return this._libp2p.peerId } - // handle messages received through the network + /** + * handle messages received through the network + * + * @param {PeerId} peerId + * @param {BitswapMessage} incoming + * @returns {Promise} + */ async _receiveMessage (peerId, incoming) { try { // Note: this allows the engine to respond to any wants in the message. @@ -99,6 +112,13 @@ class Bitswap { })) } + /** + * @private + * @param {PeerId} peerId + * @param {Block} block + * @param {boolean} wasWanted + * @returns {Promise} + */ async _handleReceivedBlock (peerId, block, wasWanted) { this._log('received block') @@ -113,27 +133,46 @@ class Bitswap { await this.put(block) } - _updateReceiveCounters (peerId, block, exists) { - this._stats.push(peerId, 'blocksReceived', 1) - this._stats.push(peerId, 'dataReceived', block.data.length) + /** + * @private + * @param {string} peerIdStr + * @param {Block} block + * @param {boolean} exists + */ + _updateReceiveCounters (peerIdStr, block, exists) { + this._stats.push(peerIdStr, 'blocksReceived', 1) + this._stats.push(peerIdStr, 'dataReceived', block.data.length) if (exists) { - this._stats.push(peerId, 'dupBlksReceived', 1) - this._stats.push(peerId, 'dupDataReceived', block.data.length) + this._stats.push(peerIdStr, 'dupBlksReceived', 1) + this._stats.push(peerIdStr, 'dupDataReceived', block.data.length) } } - // handle errors on the receiving channel + /** + * handle errors on the receiving channel + * + * @param {Error} err + * @returns {void} + */ _receiveError (err) { this._log.error('ReceiveError: %s', err.message) } - // handle new peers + /** + * handle new peers + * + * @param {PeerId} peerId + */ _onPeerConnected (peerId) { this.wm.connected(peerId) } - // handle peers being disconnected + /** + * handle peers being disconnected + * + * @param {PeerId} peerId + */ _onPeerDisconnected (peerId) { this.wm.disconnected(peerId) this.engine.peerDisconnected(peerId) @@ -158,7 +197,7 @@ class Bitswap { * Return the current wantlist for a given `peerId` * * @param {PeerId} peerId - * @returns {Map} + * @returns {Map} */ wantlistForPeer (peerId) { return this.engine.wantlistForPeer(peerId) @@ -179,11 +218,17 @@ class Bitswap { * blockstore it is returned, otherwise the block is added to the wantlist and returned once another node sends it to us. * * @param {CID} cid - * @param {Object} options - * @param {AbortSignal} options.abortSignal + * @param {Object} [options] + * @param {AbortSignal} [options.signal] * @returns {Promise} */ async get (cid, options = {}) { + /** + * @param {CID} cid + * @param {Object} options + * @param {AbortSignal} options.signal + * @returns {Promise} + */ const fetchFromNetwork = (cid, options) => { // add it to the want list - n.b. later we will abort the AbortSignal // so no need to remove the blocks from the wantlist after we have it @@ -194,6 +239,13 @@ class Bitswap { let promptedNetwork = false + /** + * + * @param {CID} cid + * @param {Object} options + * @param {AbortSignal} options.signal + * @returns {Promise} + */ const loadOrFetchFromNetwork = async (cid, options) => { try { // have to await here as we want to handle ERR_NOT_FOUND @@ -243,10 +295,10 @@ class Bitswap { * Fetch a a list of blocks by cid. If the blocks are in the local * blockstore they are returned, otherwise the blocks are added to the wantlist and returned once another node sends them to us. * - * @param {AsyncIterator} cids - * @param {Object} options - * @param {AbortSignal} options.abortSignal - * @returns {Promise>} + * @param {AsyncIterable|Iterable} cids + * @param {Object} [options] + * @param {AbortSignal} [options.signal] + * @returns {AsyncIterable} */ async * getMany (cids, options = {}) { for await (const cid of cids) { @@ -262,16 +314,14 @@ class Bitswap { * If you want to cancel the want for a block without doing that, pass an * AbortSignal in to `.get` or `.getMany` and abort it. * - * @param {Iterable} cids + * @param {CID[]|CID} cids * @returns {void} */ unwant (cids) { - if (!Array.isArray(cids)) { - cids = [cids] - } + const cidsArray = Array.isArray(cids) ? cids : [cids] - this.wm.unwantBlocks(cids) - cids.forEach((cid) => this.notifications.unwantBlock(cid)) + this.wm.unwantBlocks(cidsArray) + cidsArray.forEach((cid) => this.notifications.unwantBlock(cid)) } /** @@ -279,14 +329,11 @@ class Bitswap { * for blocks to never resolve. If you wish these promises to abort instead * call `unwant(cids)` instead. * - * @param {Iterable} cids + * @param {CID[]|CID} cids * @returns {void} */ cancelWants (cids) { - if (!Array.isArray(cids)) { - cids = [cids] - } - this.wm.cancelWants(cids) + this.wm.cancelWants(Array.isArray(cids) ? cids : [cids]) } /** @@ -305,7 +352,7 @@ class Bitswap { * Put the given blocks to the underlying blockstore and * send it to nodes that have it them their wantlist. * - * @param {AsyncIterable} blocks + * @param {AsyncIterable|Iterable} blocks * @returns {AsyncIterable} */ async * putMany (blocks) { @@ -319,6 +366,7 @@ class Bitswap { /** * Sends notifications about the arrival of a block * + * @private * @param {Block} block */ _sendHaveBlockNotifications (block) { @@ -333,7 +381,7 @@ class Bitswap { /** * Get the current list of wants. * - * @returns {Iterator} + * @returns {Iterable<[string, WantListEntry]>} */ getWantlist () { return this.wm.wantlist.entries() @@ -342,7 +390,7 @@ class Bitswap { /** * Get the current list of partners. * - * @returns {Iterator} + * @returns {PeerId[]} */ peers () { return this.engine.peers() @@ -351,7 +399,7 @@ class Bitswap { /** * Get stats about the bitswap node. * - * @returns {Object} + * @returns {Stats} */ stat () { return this._stats @@ -382,3 +430,13 @@ class Bitswap { } module.exports = Bitswap + +/** + * @typedef {import('./types').LibP2P} LibP2P + * @typedef {import('./types').BlockStore} BlockStore + * @typedef {import('./types').PeerId} PeerId + * @typedef {import('./types/message')} BitswapMessage + * @typedef {import('./types').Block} Block + * @typedef {import('./types').CID} CID + * @typedef {import('./types/wantlist/entry')} WantListEntry + */ diff --git a/src/network.js b/src/network.js index 77a3b2f9..1f82aaba 100644 --- a/src/network.js +++ b/src/network.js @@ -1,7 +1,7 @@ 'use strict' const lp = require('it-length-prefixed') -const pipe = require('it-pipe') +const { pipe } = require('it-pipe') const MulticodecTopology = require('libp2p-interfaces/src/topology/multicodec-topology') @@ -14,6 +14,13 @@ const BITSWAP110 = '/ipfs/bitswap/1.1.0' const BITSWAP120 = '/ipfs/bitswap/1.2.0' class Network { + /** + * @param {LibP2P} libp2p + * @param {BitSwap} bitswap + * @param {Object} options + * @param {boolean} [options.b100Only] + * @param {Stats} stats + */ constructor (libp2p, bitswap, options, stats) { this._log = logger(libp2p.peerId, 'network') options = options || {} @@ -69,12 +76,13 @@ class Network { /** * Handles both types of incoming bitswap messages + * * @private - * @param {object} param0 - * @param {string} param0.protocol The protocol the stream is running - * @param {Stream} param0.stream A duplex iterable stream - * @param {Connection} param0.connection A libp2p Connection - * @returns {void} + * @param {object} connection + * @param {string} connection.protocol - The protocol the stream is running + * @param {Stream} connection.stream - A duplex iterable stream + * @param {Connection} connection.connection - A libp2p Connection + * @returns {Promise} */ async _onConnection ({ protocol, stream, connection }) { if (!this._running) { return } @@ -101,10 +109,19 @@ class Network { } } + /** + * @private + * @param {PeerId} peerId + */ _onPeerConnect (peerId) { this.bitswap._onPeerConnected(peerId) } + /** + * @private + * @param {PeerId} peerId + * @returns {void} + */ _onPeerDisconnect (peerId) { this.bitswap._onPeerDisconnected(peerId) } @@ -114,9 +131,9 @@ class Network { * * @param {CID} cid * @param {number} maxProviders - * @param {Object} options - * @param {AbortSignal} options.abortSignal - * @returns {AsyncIterable} + * @param {Object} [options] + * @param {AbortSignal} [options.signal] + * @returns {AsyncIterable} */ findProviders (cid, maxProviders, options = {}) { return this.libp2p.contentRouting.findProviders( @@ -133,9 +150,9 @@ class Network { * Find the providers of a given `cid` and connect to them. * * @param {CID} cid - * @param {Object} options - * @param {AbortSignal} options.abortSignal - * @returns {void} + * @param {Object} [options] + * @param {AbortSignal} [options.signal] + * @returns {Promise} */ async findAndConnect (cid, options) { const connectAttempts = [] @@ -150,16 +167,22 @@ class Network { * Tell the network we can provide content for the passed CID * * @param {CID} cid - * @param {Object} options - * @param {AbortSignal} options.abortSignal + * @param {Object} [options] + * @param {AbortSignal} [options.signal] * @returns {Promise} */ async provide (cid, options) { await this.libp2p.contentRouting.provide(cid, options) } - // Connect to the given peer - // Send the given msg (instance of Message) to the given peer + /** + * Connect to the given peer + * Send the given msg (instance of Message) to the given peer + * + * @param {PeerId} peer + * @param {Message} msg + * @returns {Promise} + */ async sendMessage (peer, msg) { if (!this._running) throw new Error('network isn\'t running') @@ -168,6 +191,7 @@ class Network { const { stream, protocol } = await this._dialPeer(peer) + /** @type {Uint8Array} */ let serialized switch (protocol) { case BITSWAP100: @@ -190,9 +214,9 @@ class Network { /** * Connects to another peer * - * @param {PeerId|Multiaddr} peer - * @param {Object} options - * @param {AbortSignal} options.abortSignal + * @param {PeerId|Multiaddr|Provider} peer + * @param {Object} [options] + * @param {AbortSignal} [options.signal] * @returns {Promise} */ async connectTo (peer, options) { // eslint-disable-line require-await @@ -203,11 +227,21 @@ class Network { return this.libp2p.dial(peer, options) } - // Dial to the peer and try to use the most recent Bitswap + /** + * Dial to the peer and try to use the most recent Bitswap + * + * @private + * @param {PeerId|Multiaddr|Provider} peer + */ _dialPeer (peer) { return this.libp2p.dialProtocol(peer, [BITSWAP120, BITSWAP110, BITSWAP100]) } + /** + * @private + * @param {PeerId} peer + * @param {Map} blocks + */ _updateSentStats (peer, blocks) { const peerId = peer.toB58String() @@ -218,6 +252,12 @@ class Network { } } +/** + * + * @param {Stream} stream + * @param {Uint8Array} msg + * @param {*} log + */ async function writeMessage (stream, msg, log) { try { await pipe( @@ -231,3 +271,24 @@ async function writeMessage (stream, msg, log) { } module.exports = Network + +/** + * @typedef {import('./types').PeerId} PeerId + * @typedef {import('./types').CID} CID + * @typedef {import('./types').Multiaddr} Multiaddr + * @typedef {import('./types').LibP2P} LibP2P + * @typedef {import('./stats')} Stats + * @typedef {import('./index')} BitSwap + * + * @typedef {Object} Connection + * @property {string} id + * @property {PeerId} remotePeer + * + * @typedef {Object} Provider + * @property {PeerId} id + * @property {Multiaddr[]} multiaddrs + * + * @typedef {Object} Stream + * @property {AsyncIterable} source + * @property {(output:AsyncIterable) => Promise} sink + */ diff --git a/src/notifications.js b/src/notifications.js index 2243fec3..b6f98c4c 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -1,6 +1,6 @@ 'use strict' -const EventEmitter = require('events').EventEmitter +const { EventEmitter } = require('events') const Block = require('ipld-block') const uint8ArrayEquals = require('uint8arrays/equals') const uint8ArrayToString = require('uint8arrays/to-string') @@ -8,17 +8,23 @@ const uint8ArrayToString = require('uint8arrays/to-string') const CONSTANTS = require('./constants') const logger = require('./utils').logger +/** + * @param {CID} cid + */ const unwantEvent = (cid) => `unwant:${uint8ArrayToString(cid.multihash, 'base64')}` -const blockEvent = (cid) => `block:${uint8ArrayToString(cid.multihash, 'base64')}` /** - * Internal module used to track events about incoming blocks, - * wants and unwants. - * - * @param {PeerId} peerId - * @private + * @param {CID} cid */ +const blockEvent = (cid) => `block:${uint8ArrayToString(cid.multihash, 'base64')}` + class Notifications extends EventEmitter { + /** + * Internal module used to track events about incoming blocks, + * wants and unwants. + * + * @param {PeerId} peerId + */ constructor (peerId) { super() @@ -31,7 +37,7 @@ class Notifications extends EventEmitter { * Signal the system that we received `block`. * * @param {Block} block - * @return {void} + * @returns {void} */ hasBlock (block) { const event = blockEvent(block.cid) @@ -46,8 +52,8 @@ class Notifications extends EventEmitter { * or undefined when the block is unwanted. * * @param {CID} cid - * @param {Object} options - * @param {AbortSignal} options.abortSignal + * @param {Object} [options] + * @param {AbortSignal} [options.signal] * @returns {Promise} */ wantBlock (cid, options = {}) { @@ -107,3 +113,8 @@ class Notifications extends EventEmitter { } module.exports = Notifications + +/** + * @typedef {import('./types').CID} CID + * @typedef {import('./types').PeerId} PeerId + */ diff --git a/src/stats/index.js b/src/stats/index.js index a7a9e90f..829d05c1 100644 --- a/src/stats/index.js +++ b/src/stats/index.js @@ -1,17 +1,28 @@ 'use strict' -const EventEmitter = require('events') +const { EventEmitter } = require('events') const Stat = require('./stat') +/** + * @typedef {[number, number, number]} AvarageIntervals + */ const defaultOptions = { - movingAverageIntervals: [ + movingAverageIntervals: /** @type {AvarageIntervals} */ ([ 60 * 1000, // 1 minute 5 * 60 * 1000, // 5 minutes 15 * 60 * 1000 // 15 minutes - ] + ]) } class Stats extends EventEmitter { + /** + * + * @param {*} initialCounters + * @param {Object} _options + * @param {boolean} _options.enabled + * @param {number} _options.computeThrottleTimeout + * @param {number} _options.computeThrottleMaxQueueSize + */ constructor (initialCounters, _options) { super() @@ -32,6 +43,7 @@ class Stats extends EventEmitter { this._global = new Stat(initialCounters, options) this._global.on('update', (stats) => this.emit('update', stats)) + /** @type {Map} */ this._peers = new Map() } @@ -63,14 +75,24 @@ class Stats extends EventEmitter { return this._global.movingAverages } + /** + * @param {PeerId|string} peerId + * @returns {Stat|void} + */ forPeer (peerId) { - if (peerId.toB58String) { - peerId = peerId.toB58String() - } + const peerIdStr = (typeof peerId !== 'string' && peerId.toB58String) + ? peerId.toB58String() + : `${peerId}` - return this._peers.get(peerId) + return this._peers.get(peerIdStr) } + /** + * + * @param {string|null} peer + * @param {string} counter + * @param {number} inc + */ push (peer, counter, inc) { if (this._enabled) { this._global.push(counter, inc) @@ -98,3 +120,7 @@ class Stats extends EventEmitter { } module.exports = Stats + +/** + * @typedef {import('../types').PeerId} PeerId + */ diff --git a/src/stats/stat.js b/src/stats/stat.js index da3e461e..7acbeaa7 100644 --- a/src/stats/stat.js +++ b/src/stats/stat.js @@ -1,25 +1,38 @@ 'use strict' -const EventEmitter = require('events') -const Big = require('bignumber.js') +const { EventEmitter } = require('events') +const Big = require('bignumber.js').default const MovingAverage = require('moving-average') class Stats extends EventEmitter { + /** + * + * @param {string[]} initialCounters + * @param {Object} options + * @param {boolean} options.enabled + * @param {number} options.computeThrottleTimeout + * @param {number} options.computeThrottleMaxQueueSize + * @param {import('.').AvarageIntervals} options.movingAverageIntervals + */ constructor (initialCounters, options) { super() this._options = options + /** @type {Op[]} */ this._queue = [] + /** @type {Record} */ this._stats = {} this._frequencyLastTime = Date.now() this._frequencyAccumulators = {} + + /** @type {Record>} */ this._movingAverages = {} this._update = this._update.bind(this) initialCounters.forEach((key) => { - this._stats[key] = Big(0) + this._stats[key] = new Big(0) this._movingAverages[key] = {} this._options.movingAverageIntervals.forEach((interval) => { const ma = this._movingAverages[key][interval] = MovingAverage(interval) @@ -52,6 +65,10 @@ class Stats extends EventEmitter { return Object.assign({}, this._movingAverages) } + /** + * @param {string} counter + * @param {number} inc + */ push (counter, inc) { if (this._enabled) { this._queue.push([counter, inc, Date.now()]) @@ -59,6 +76,9 @@ class Stats extends EventEmitter { } } + /** + * @private + */ _resetComputeTimeout () { if (this._timeout) { clearTimeout(this._timeout) @@ -66,12 +86,19 @@ class Stats extends EventEmitter { this._timeout = setTimeout(this._update, this._nextTimeout()) } + /** + * @private + * @returns {number} + */ _nextTimeout () { // calculate the need for an update, depending on the queue length const urgency = this._queue.length / this._options.computeThrottleMaxQueueSize return Math.max(this._options.computeThrottleTimeout * (1 - urgency), 0) } + /** + * @private + */ _update () { this._timeout = null @@ -79,15 +106,19 @@ class Stats extends EventEmitter { let last while (this._queue.length) { const op = last = this._queue.shift() - this._applyOp(op) + op && this._applyOp(op) } - this._updateFrequency(last[2]) // contains timestamp of last op + last && this._updateFrequency(last[2]) // contains timestamp of last op this.emit('update', this._stats) } } + /** + * @private + * @param {number} latestTime + */ _updateFrequency (latestTime) { const timeDiff = latestTime - this._frequencyLastTime @@ -100,6 +131,13 @@ class Stats extends EventEmitter { this._frequencyLastTime = latestTime } + /** + * @private + * @param {string} key + * @param {number} timeDiffMS + * @param {number} latestTime + * @returns {void} + */ _updateFrequencyFor (key, timeDiffMS, latestTime) { const count = this._frequencyAccumulators[key] || 0 this._frequencyAccumulators[key] = 0 @@ -118,18 +156,22 @@ class Stats extends EventEmitter { }) } + /** + * @private + * @param {Op} op + */ _applyOp (op) { const key = op[0] const inc = op[1] if (typeof inc !== 'number') { - throw new Error('invalid increment number:', inc) + throw new Error(`invalid increment number: ${inc}`) } let n if (!Object.prototype.hasOwnProperty.call(this._stats, key)) { - n = this._stats[key] = Big(0) + n = this._stats[key] = new Big(0) } else { n = this._stats[key] } @@ -143,3 +185,7 @@ class Stats extends EventEmitter { } module.exports = Stats + +/** + * @typedef {[string, number, number]} Op + */ diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 00000000..9375d7c3 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,64 @@ +import CID from 'cids' +import PeerId from 'peer-id' +import Multiaddr from 'multiaddr' +import LibP2P from 'libp2p' +import Block from 'ipld-block' + +export { CID, PeerId, Multiaddr, Block, LibP2P } + +export type WantBlock = 0 +export type HaveBlock = 1 +export type WantType = WantBlock | HaveBlock + +export type BlockData = { + prefix: Uint8Array + data: Uint8Array +} + +export type Have = 0 +export type DontHave = 1 +export type BlockPresenceType = Have | DontHave + +export type BlockPresence = { + cid: Uint8Array + type: BlockPresenceType +} + +export type Entry = { + block: Uint8Array + priority: number + cancel: boolean + wantType?: WantType + sendDontHave?: boolean +} + +export type Message110 = { + wantlist: WantList + blockPresences: BlockPresence[] + payload: BlockData[] + + pendingBytes?: number +} + +export type Message100 = { + wantlist: WantList + blocks: Uint8Array[] + + pendingBytes?: number +} + +export type WantList = { + entries: Entry[] + full?: boolean +} + +export type AbortOptions = { + signal: AbortSignal +} + +export interface BlockStore { + has(cid:CID, options?:AbortOptions):Promise + get(cid:CID, options?:AbortOptions):Promise + put(block:Block, options?:AbortOptions):Promise + putMany(blocks:AsyncIterable|Iterable, options?:AbortOptions):AsyncIterable +} diff --git a/src/types/message/entry.js b/src/types/message/entry.js index c488ea84..2bcb0606 100644 --- a/src/types/message/entry.js +++ b/src/types/message/entry.js @@ -3,6 +3,13 @@ const WantlistEntry = require('../wantlist').Entry module.exports = class BitswapMessageEntry { + /** + * @param {CID} cid + * @param {number} priority + * @param {WantType} wantType + * @param {boolean} [cancel] + * @param {boolean} [sendDontHave] + */ constructor (cid, priority, wantType, cancel, sendDontHave) { this.entry = new WantlistEntry(cid, priority, wantType) this.cancel = Boolean(cancel) @@ -45,3 +52,8 @@ module.exports = class BitswapMessageEntry { this.entry.equals(other.entry) } } + +/** + * @typedef {import('../../types').WantType} WantType + * @typedef {import('../../types').CID} CID + */ diff --git a/src/types/message/index.js b/src/types/message/index.js index c4be0c58..9f780ab1 100644 --- a/src/types/message/index.js +++ b/src/types/message/index.js @@ -10,10 +10,19 @@ const { Message } = require('./message.proto') const Entry = require('./entry') class BitswapMessage { + /** + * @param {boolean} full + */ constructor (full) { this.full = full + /** @type {Map} */ this.wantlist = new Map() + + /** @type {Map} + */ this.blocks = new Map() + + /** @type {Map} */ this.blockPresences = new Map() this.pendingBytes = 0 } @@ -24,6 +33,15 @@ class BitswapMessage { this.blockPresences.size === 0 } + /** + * + * @param {CID} cid + * @param {number} priority + * @param {WantType} [wantType] + * @param {boolean} [cancel] + * @param {boolean} [sendDontHave] + * @returns {void} + */ addEntry (cid, priority, wantType, cancel, sendDontHave) { if (wantType == null) { wantType = BitswapMessage.WantType.Block @@ -53,11 +71,18 @@ class BitswapMessage { } } + /** + * @param {Block} block + * @returns {void} + */ addBlock (block) { const cidStr = block.cid.toString('base58btc') this.blocks.set(cidStr, block) } + /** + * @param {CID} cid + */ addHave (cid) { const cidStr = cid.toString('base58btc') if (!this.blockPresences.has(cidStr)) { @@ -65,6 +90,9 @@ class BitswapMessage { } } + /** + * @param {CID} cid + */ addDontHave (cid) { const cidStr = cid.toString('base58btc') if (!this.blockPresences.has(cidStr)) { @@ -72,21 +100,30 @@ class BitswapMessage { } } + /** + * @param {CID} cid + */ cancel (cid) { const cidStr = cid.toString('base58btc') this.wantlist.delete(cidStr) this.addEntry(cid, 0, BitswapMessage.WantType.Block, true, false) } + /** + * @param {number} size + */ setPendingBytes (size) { this.pendingBytes = size } - /* + /** * Serializes to Bitswap Message protobuf of * version 1.0.0 + * + * @returns {Uint8Array} */ serializeToBitswap100 () { + /** @type {Message100} */ const msg = { wantlist: { entries: Array.from(this.wantlist.values()).map((entry) => { @@ -108,11 +145,14 @@ class BitswapMessage { return Message.encode(msg) } - /* + /** * Serializes to Bitswap Message protobuf of * version 1.1.0 + * + * @returns {Uint8Array} */ serializeToBitswap110 () { + /** @type {Message110} */ const msg = { wantlist: { entries: Array.from(this.wantlist.values()).map((entry) => { @@ -154,11 +194,18 @@ class BitswapMessage { return Message.encode(msg) } + /** + * @param {BitswapMessage} other + * @returns {boolean} + */ equals (other) { if (this.full !== other.full || this.pendingBytes !== other.pendingBytes || !isMapEqual(this.wantlist, other.wantlist) || !isMapEqual(this.blocks, other.blocks) || + // @TODO - Is this a bug ? `isMapEqual` only compare maps of blocks and + // wants + // @ts-ignore !isMapEqual(this.blockPresences, other.blockPresences) ) { return false @@ -174,6 +221,11 @@ class BitswapMessage { } } +/** + * + * @param {Uint8Array} raw + * @returns {Promise} + */ BitswapMessage.deserialize = async (raw) => { const decoded = Message.decode(raw) @@ -232,6 +284,9 @@ BitswapMessage.deserialize = async (raw) => { return msg } +/** + * @param {CID} cid + */ BitswapMessage.blockPresenceSize = (cid) => { // It's ok if this is not exactly right: it's used to estimate the size of // the HAVE / DONT_HAVE on the wire, but when doing that calculation we leave @@ -242,11 +297,22 @@ BitswapMessage.blockPresenceSize = (cid) => { BitswapMessage.Entry = Entry BitswapMessage.WantType = { + /** @type {import('../../types').WantBlock} */ Block: Message.Wantlist.WantType.Block, + /** @type {import('../../types').HaveBlock} */ Have: Message.Wantlist.WantType.Have } BitswapMessage.BlockPresenceType = { + /** @type {import('../../types').Have} */ Have: Message.BlockPresenceType.Have, + /** @type {import('../../types').DontHave} */ DontHave: Message.BlockPresenceType.DontHave } module.exports = BitswapMessage + +/** + * @typedef {import('../../types').WantType} WantType + * @typedef {import('../../types').Message110} Message110 + * @typedef {import('../../types').Message100} Message100 + * @typedef {import('../../types').BlockPresenceType} BlockPresenceType + */ diff --git a/src/types/wantlist/entry.js b/src/types/wantlist/entry.js index 84d68826..e65db309 100644 --- a/src/types/wantlist/entry.js +++ b/src/types/wantlist/entry.js @@ -1,6 +1,11 @@ 'use strict' class WantListEntry { + /** + * @param {CID} cid + * @param {number} priority + * @param {WantType} [wantType] + */ constructor (cid, priority, wantType) { // Keep track of how many requests we have for this key this._refCounter = 1 @@ -28,6 +33,10 @@ class WantListEntry { return `WantlistEntry ` } + /** + * @param {WantListEntry} other + * @returns {boolean} + */ equals (other) { return (this._refCounter === other._refCounter) && this.cid.equals(other.cid) && @@ -37,3 +46,8 @@ class WantListEntry { } module.exports = WantListEntry + +/** + * @typedef {import('../../types').WantType} WantType + * @typedef {import('../../types').CID} CID + */ diff --git a/src/types/wantlist/index.js b/src/types/wantlist/index.js index 3c4656b7..4a527e7c 100644 --- a/src/types/wantlist/index.js +++ b/src/types/wantlist/index.js @@ -4,7 +4,12 @@ const { sortBy } = require('../../utils') const Entry = require('./entry') class Wantlist { + /** + * + * @param {Stats} [stats] + */ constructor (stats) { + /** @type {Map} */ this.set = new Map() this._stats = stats } @@ -13,6 +18,12 @@ class Wantlist { return this.set.size } + /** + * @param {CID} cid + * @param {number} priority + * @param {WantType} [wantType] + * @returns {void} + */ add (cid, priority, wantType) { // Have to import here to avoid circular reference const Message = require('../message') @@ -36,6 +47,10 @@ class Wantlist { } } + /** + * @param {CID} cid + * @returns {void} + */ remove (cid) { const cidStr = cid.toString('base58btc') const entry = this.set.get(cidStr) @@ -57,12 +72,18 @@ class Wantlist { } } + /** + * @param {string} cidStr + */ removeForce (cidStr) { if (this.set.has(cidStr)) { this.set.delete(cidStr) } } + /** + * @param {(entry:Entry, key:string) => void} fn + */ forEach (fn) { return this.set.forEach(fn) } @@ -72,9 +93,16 @@ class Wantlist { } sortedEntries () { + // @ts-ignore - Property 'key' does not exist on type 'WantListEntry'.ts(2339) + // TODO: Figure out if this is an actual bug. return new Map(sortBy(o => o[1].key, Array.from(this.set.entries()))) } + /** + * + * @param {CID} cid + * @returns {Entry|undefined} + */ contains (cid) { const cidStr = cid.toString('base58btc') return this.set.get(cidStr) @@ -83,3 +111,9 @@ class Wantlist { Wantlist.Entry = Entry module.exports = Wantlist + +/** + * @typedef {import('../../types').CID} CID + * @typedef {import('../../types').WantType} WantType + * @typedef {import('../../stats')} Stats + */ diff --git a/src/utils/index.js b/src/utils/index.js index 334bd17a..138eb315 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -8,9 +8,6 @@ const uint8ArrayEquals = require('uint8arrays/equals') * * @param {PeerId} [id] * @param {string} [subsystem] - * @returns {debug} - * - * @private */ const logger = (id, subsystem) => { const name = ['bitswap'] @@ -20,12 +17,19 @@ const logger = (id, subsystem) => { if (id) { name.push(`${id.toB58String().slice(0, 8)}`) } - const logger = debug(name.join(':')) - logger.error = debug(name.concat(['error']).join(':')) - return logger + return Object.assign(debug(name.join(':')), { + error: debug(name.concat(['error']).join(':')) + }) } +/** + * @template X, T + * @param {(x:X, t:T) => boolean} pred + * @param {X} x + * @param {T[]} list + * @returns {boolean} + */ const includesWith = (pred, x, list) => { let idx = 0 const len = list.length @@ -38,6 +42,12 @@ const includesWith = (pred, x, list) => { return false } +/** + * @template T + * @param {(x:T, t:T) => boolean} pred + * @param {T[]} list + * @returns {T[]} + */ const uniqWith = (pred, list) => { let idx = 0 const len = list.length @@ -54,6 +64,13 @@ const uniqWith = (pred, list) => { return result } +/** + * @template {string|number|symbol} K + * @template V + * @param {(v:V) => K} pred + * @param {V[]} list + * @returns {Record} + */ const groupBy = (pred, list) => { return list.reduce((acc, v) => { const k = pred(v) @@ -64,15 +81,28 @@ const groupBy = (pred, list) => { acc[k] = [v] } return acc - }, {}) + }, /** @type {Record} */({})) } +/** + * @template T, E + * @param {(a:T, b:E) => boolean} pred + * @param {T[]} list + * @param {E[]} values + * @returns {T[]} + */ const pullAllWith = (pred, list, values) => { return list.filter(i => { return !includesWith(pred, i, values) }) } +/** + * @template T + * @param {(v:T) => number} fn + * @param {T[]} list + * @returns {T[]} + */ const sortBy = (fn, list) => { return Array.prototype.slice.call(list, 0).sort((a, b) => { const aa = fn(a) @@ -83,8 +113,10 @@ const sortBy = (fn, list) => { /** * Is equal for Maps of BitswapMessageEntry or Blocks - * @param {Map} a - * @param {Map} b + * + * @template {{data?:Uint8Array, equals?: (value:any) => boolean}} T + * @param {Map} a + * @param {Map} b * @returns {boolean} */ const isMapEqual = (a, b) => { @@ -93,18 +125,18 @@ const isMapEqual = (a, b) => { } for (const [key, valueA] of a) { - if (!b.has(key)) { + const valueB = b.get(key) + + if (valueB === undefined) { return false } - const valueB = b.get(key) - // Support BitswapMessageEntry if (typeof valueA.equals === 'function' && !valueA.equals(valueB)) { return false } // Support Blocks - if (valueA._data && !uint8ArrayEquals(valueA._data, valueB._data)) { + if (valueA.data && !uint8ArrayEquals(valueA.data, valueB.data)) { return false } } @@ -121,3 +153,8 @@ module.exports = { sortBy, isMapEqual } + +/** + * @typedef {import('../types').PeerId} PeerId + * @typedef {import('../types/message')} BitswapMessageEntry + */ diff --git a/src/utils/sorted-map.js b/src/utils/sorted-map.js index 61688170..d79315c2 100644 --- a/src/utils/sorted-map.js +++ b/src/utils/sorted-map.js @@ -1,16 +1,19 @@ 'use strict' /** + * @template Key, Value * SortedMap is a Map whose iterator order can be defined by the user + * @extends {Map} */ class SortedMap extends Map { /** - * @param {Array} [entries] - * @param {function(a, b)} [cmp] compares [k1, v1] to [k2, v2] + * @param {Array<[Key, Value]>} [entries] + * @param {(a:[Key, Value], b:[Key, Value]) => number} [cmp] - compares [k1, v1] to [k2, v2] */ constructor (entries, cmp) { super() this._cmp = cmp || this._defaultSort + /** @type {Key[]} */ this._keys = [] for (const [k, v] of entries || []) { this.set(k, v) @@ -23,7 +26,8 @@ class SortedMap extends Map { * priority changes, call update. * Call indexOf() to get the index _before_ the change happens. * - * @param {Object} i - the index of entry whose position should be updated. + * @param {number} i - the index of entry whose position should be updated. + * @returns {void} */ update (i) { if (i < 0 || i >= this._keys.length) { @@ -36,6 +40,10 @@ class SortedMap extends Map { this._keys.splice(newIdx, 0, k) } + /** + * @param {Key} k + * @param {Value} v + */ set (k, v) { // If the key is already in the map, remove it from the ordering and // re-insert it below @@ -50,6 +58,8 @@ class SortedMap extends Map { // Find the correct position of the newly inserted k/v in the order const i = this._find(k) this._keys.splice(i, 0, k) + + return this } clear () { @@ -57,16 +67,23 @@ class SortedMap extends Map { this._keys = [] } + /** + * @param {Key} k + */ delete (k) { if (!this.has(k)) { - return + return false } const i = this.indexOf(k) this._keys.splice(i, 1) - super.delete(k) + return super.delete(k) } + /** + * @param {Key} k + * @returns {number} + */ indexOf (k) { if (!this.has(k)) { return -1 @@ -88,6 +105,12 @@ class SortedMap extends Map { return -1 // should never happen for existing key } + /** + * @private + * @param {Key} k + * @returns {number} + */ + _find (k) { let lower = 0 let upper = this._keys.length @@ -106,22 +129,43 @@ class SortedMap extends Map { return lower } + /** + * @returns {IterableIterator} + */ * keys () { for (const k of this._keys) { yield k } + + return undefined } + /** + * @returns {IterableIterator} + */ * values () { for (const k of this._keys) { + // @ts-ignore - return of `this.get(k)` is `Value|undefined` which is + // incompatible with `Value`. Typechecker can't that this contains values + // for all the `_keys`. ts(2322) yield this.get(k) } + + return undefined } + /** + * @returns {IterableIterator<[Key, Value]>} + */ * entries () { for (const k of this._keys) { + // @ts-ignore - return of `this.get(k)` is `Value|undefined` which is + // incompatible with `Value`. Typechecker can't that this contains values + // for all the `_keys`. ts(2322) yield [k, this.get(k)] } + + return undefined } * [Symbol.iterator] () { @@ -138,15 +182,29 @@ class SortedMap extends Map { } } + /** + * @private + * @param {[Key, Value]} a + * @param {[Key, Value]} b + * @returns {0|1|-1} + */ _defaultSort (a, b) { if (a[0] < b[0]) return -1 if (b[0] < a[0]) return 1 return 0 } + /** + * @private + * @param {Key} a + * @param {Key} b + * @returns {number} + */ _kCmp (a, b) { return this._cmp( + // @ts-ignore - get may return undefined [a, this.get(a)], + // @ts-ignore - get may return undefined [b, this.get(b)] ) } diff --git a/src/want-manager/index.js b/src/want-manager/index.js index 2e9edfba..9df65099 100644 --- a/src/want-manager/index.js +++ b/src/want-manager/index.js @@ -7,6 +7,12 @@ const MsgQueue = require('./msg-queue') const logger = require('../utils').logger module.exports = class WantManager { + /** + * + * @param {PeerId} peerId + * @param {Network} network + * @param {Stats} stats + */ constructor (peerId, network, stats) { this.peers = new Map() this.wantlist = new Wantlist(stats) @@ -122,7 +128,11 @@ module.exports = class WantManager { stop () { this.peers.forEach((mq) => this.disconnected(mq.peerId)) - - clearInterval(this.timer) } } + +/** + * @typedef {import('../types').PeerId} PeerId + * @typedef {import('../network')} Network + * @typedef {import('../stats')} Stats + */ diff --git a/src/want-manager/msg-queue.js b/src/want-manager/msg-queue.js index 54d1e58e..3a50412c 100644 --- a/src/want-manager/msg-queue.js +++ b/src/want-manager/msg-queue.js @@ -7,13 +7,19 @@ const logger = require('../utils').logger const { wantlistSendDebounceMs } = require('../constants') module.exports = class MsgQueue { + /** + * + * @param {PeerId} selfPeerId + * @param {PeerId} otherPeerId + * @param {Network} network + */ constructor (selfPeerId, otherPeerId, network) { this.peerId = otherPeerId this.network = network this.refcnt = 1 this._entries = [] - this._log = logger(selfPeerId, 'msgqueue', otherPeerId.toB58String().slice(0, 8)) + this._log = logger(selfPeerId, 'msgqueue') this.sendEntries = debounce(this._sendEntries.bind(this), wantlistSendDebounceMs) } @@ -63,3 +69,8 @@ module.exports = class MsgQueue { }) } } + +/** + * @typedef {import('../types').PeerId} PeerId + * @typedef {import('../network')} Network + */ diff --git a/test/network/gen-bitswap-network.node.js b/test/network/gen-bitswap-network.node.js index 89a7a0bc..bb71451d 100644 --- a/test/network/gen-bitswap-network.node.js +++ b/test/network/gen-bitswap-network.node.js @@ -55,8 +55,8 @@ describe('gen Bitswap network', function () { /** * @private - * @param {Array<*>} nodes Array of Bitswap Network nodes - * @param {number} blocksPerNode Number of blocks to exchange per node + * @param {Array<*>} nodes - Array of Bitswap Network nodes + * @param {number} blocksPerNode - Number of blocks to exchange per node */ async function exchangeBlocks (nodes, blocksPerNode = 10) { const blocks = await createBlocks(nodes.length * blocksPerNode) @@ -88,8 +88,9 @@ async function exchangeBlocks (nodes, blocksPerNode = 10) { /** * Resolves `num` blocks + * * @private - * @param {number} num The number of blocks to create + * @param {number} num - The number of blocks to create * @returns {Promise} */ function createBlocks (num) { diff --git a/test/utils/mocks.js b/test/utils/mocks.js index 696e608a..7dab93de 100644 --- a/test/utils/mocks.js +++ b/test/utils/mocks.js @@ -150,8 +150,8 @@ exports.applyNetwork = (bs, n) => { /** * @private - * @param {number} n The number of nodes in the network - * @param {boolean} enableDHT Whether or not to run the dht + * @param {number} n - The number of nodes in the network + * @param {boolean} enableDHT - Whether or not to run the dht */ exports.genBitswapNetwork = async (n, enableDHT = false) => { const netArray = [] // bitswap, peerStore, libp2p, peerId, repo diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000..4b69be3c --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,42 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": false, + "noImplicitAny": false, + "noImplicitThis": true, + "noFallthroughCasesInSwitch": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "strictFunctionTypes": false, + "strictNullChecks": true, + "strictPropertyInitialization": true, + "strictBindCallApply": true, + "strict": true, + "alwaysStrict": true, + "esModuleInterop": true, + "target": "ES2018", + "moduleResolution": "node", + "declaration": true, + "declarationMap": true, + "outDir": "dist", + "skipLibCheck": true, + "stripInternal": true, + "resolveJsonModule": true, + "paths": { + "multiformats": [ + "src" + ] + }, + "baseUrl": "." + }, + "include": [ + "src" + ], + "exclude": [ + "vendor", + "node_modules" + ], + "compileOnSave": false +} From 0ca4e3d5dcf528503277670b26cd4ccc92c01e41 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 13 Oct 2020 23:52:52 -0700 Subject: [PATCH 02/13] fix: fill type holes --- src/decision-engine/index.js | 3 ++- src/stats/index.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/decision-engine/index.js b/src/decision-engine/index.js index a7b7f7e8..eaaa375f 100644 --- a/src/decision-engine/index.js +++ b/src/decision-engine/index.js @@ -29,7 +29,7 @@ class DecisionEngine { /** * * @param {PeerId} peerId - * @param {*} blockstore + * @param {BlockStore} blockstore * @param {import('../network')} network * @param {Stats} stats * @param {Object} [opts] @@ -485,4 +485,5 @@ module.exports = DecisionEngine * @typedef {import('../types').Block} Block * @typedef {import('../types/message/entry')} BitswapMessageEntry * @typedef {import('../types/wantlist/entry')} WantListEntry + * @typedef {import('../types').BlockStore} BlockStore */ diff --git a/src/stats/index.js b/src/stats/index.js index 829d05c1..3dde9066 100644 --- a/src/stats/index.js +++ b/src/stats/index.js @@ -17,7 +17,7 @@ const defaultOptions = { class Stats extends EventEmitter { /** * - * @param {*} initialCounters + * @param {string[]} initialCounters * @param {Object} _options * @param {boolean} _options.enabled * @param {number} _options.computeThrottleTimeout From c028b3c14aeedce60c758ee92132ddb157c41974 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Fri, 27 Nov 2020 08:47:26 -0800 Subject: [PATCH 03/13] chore: address review comments --- .eslintrc | 3 --- package.json | 27 +++++++++++---------- src/decision-engine/interface.ts | 2 +- src/index.js | 6 +++-- src/stats/index.js | 4 +-- src/stats/stat.js | 2 +- src/types.ts | 15 ++++++++++++ src/types/message/message.proto.js | 1 + tsconfig.json | 39 +++--------------------------- 9 files changed, 41 insertions(+), 58 deletions(-) delete mode 100644 .eslintrc diff --git a/.eslintrc b/.eslintrc deleted file mode 100644 index ea565dd6..00000000 --- a/.eslintrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "ipfs" -} diff --git a/package.json b/package.json index 11e2ee0c..39b9da23 100644 --- a/package.json +++ b/package.json @@ -8,13 +8,18 @@ "./test/utils/create-libp2p-node": false, "./test/utils/create-temp-repo-nodejs.js": "./test/utils/create-temp-repo-browser.js" }, + "types": "dist/src/index.d.ts", "typesVersions": { "*": { - "*": [ - "dist/*" + "src/*": [ + "dist/src/*", + "dist/src/*/index" ] } }, + "eslintConfig": { + "extends": "ipfs" + }, "files": [ "dist", "src" @@ -24,18 +29,15 @@ "test:browser": "aegir test -t browser -t webworker", "test:node": "aegir test -t node", "lint": "aegir lint", - "check": "tsc --noEmit", + "check": "aegir ts -p check", "release": "aegir release", "release-minor": "aegir release --type minor", "release-major": "aegir release --type major", "bench": "node benchmarks/index", - "build": "npm run build:js & npm run build:types", - "build:js": "aegir build", - "build:types": "tsc --emitDeclarationOnly --declarationDir dist", + "build": "aegir build", "coverage": "aegir coverage --provider codecov", "docs": "aegir docs", - "benchmarks": "node test/benchmarks/get-many", - "prepare": "npm run build:types" + "benchmarks": "node test/benchmarks/get-many" }, "repository": { "type": "git", @@ -54,10 +56,10 @@ "homepage": "https://github.com/ipfs/js-ipfs-bitswap#readme", "devDependencies": { "@nodeutils/defaults-deep": "^1.1.0", - "aegir": "^27.0.0", + "aegir": "^29.1.0", "benchmark": "^2.1.4", "delay": "^4.3.0", - "ipfs-repo": "^6.0.1", + "ipfs-repo": "^7.0.0", "ipfs-utils": "^3.0.0", "iso-random-stream": "^1.1.1", "it-all": "^1.0.2", @@ -81,16 +83,15 @@ "sinon": "^9.0.0", "stats-lite": "^2.2.0", "uuid": "^8.0.0", - "typescript": "^4.0.3", "@types/debug": "^4.1.5" }, "dependencies": { "abort-controller": "^3.0.0", - "any-signal": "^1.1.0", + "any-signal": "^2.1.1", "bignumber.js": "^9.0.0", "cids": "^1.0.0", "debug": "^4.2.0", - "ipld-block": "git://github.com/ipld/js-ipld-block#typegen", + "ipld-block": "^0.11.0", "it-length-prefixed": "^3.0.0", "it-pipe": "^1.1.0", "just-debounce-it": "^1.1.0", diff --git a/src/decision-engine/interface.ts b/src/decision-engine/interface.ts index d318d1df..e9ab8608 100644 --- a/src/decision-engine/interface.ts +++ b/src/decision-engine/interface.ts @@ -1,6 +1,6 @@ export interface TaskMerger { /** - * Ggiven the existing tasks with the same topic, does the task add some new + * Given the existing tasks with the same topic, does the task add some new * information? Used to decide whether to merge the task or ignore it. */ hasNewInfo (task:Task, tasksWithTopic:Task[]): boolean diff --git a/src/index.js b/src/index.js index d27869a2..f401deae 100644 --- a/src/index.js +++ b/src/index.js @@ -7,7 +7,7 @@ const Notifications = require('./notifications') const logger = require('./utils').logger const Stats = require('./stats') const AbortController = require('abort-controller').default -const anySignal = require('any-signal') +const { anySignal } = require('any-signal') const defaultOptions = { statsEnabled: false, @@ -274,7 +274,9 @@ class Bitswap { // a race condition, so register for incoming block notifications as well // as trying to get it from the datastore const controller = new AbortController() - const signal = anySignal([options.signal, controller.signal]) + const signal = options.signal + ? anySignal([options.signal, controller.signal]) + : controller.signal const block = await Promise.race([ this.notifications.wantBlock(cid, { diff --git a/src/stats/index.js b/src/stats/index.js index 3dde9066..7e2531e4 100644 --- a/src/stats/index.js +++ b/src/stats/index.js @@ -4,10 +4,10 @@ const { EventEmitter } = require('events') const Stat = require('./stat') /** - * @typedef {[number, number, number]} AvarageIntervals + * @typedef {[number, number, number]} AverageIntervals */ const defaultOptions = { - movingAverageIntervals: /** @type {AvarageIntervals} */ ([ + movingAverageIntervals: /** @type {AverageIntervals} */ ([ 60 * 1000, // 1 minute 5 * 60 * 1000, // 5 minutes 15 * 60 * 1000 // 15 minutes diff --git a/src/stats/stat.js b/src/stats/stat.js index 7acbeaa7..272499b0 100644 --- a/src/stats/stat.js +++ b/src/stats/stat.js @@ -12,7 +12,7 @@ class Stats extends EventEmitter { * @param {boolean} options.enabled * @param {number} options.computeThrottleTimeout * @param {number} options.computeThrottleMaxQueueSize - * @param {import('.').AvarageIntervals} options.movingAverageIntervals + * @param {import('.').AverageIntervals} options.movingAverageIntervals */ constructor (initialCounters, options) { super() diff --git a/src/types.ts b/src/types.ts index 9375d7c3..cc23f443 100644 --- a/src/types.ts +++ b/src/types.ts @@ -62,3 +62,18 @@ export interface BlockStore { put(block:Block, options?:AbortOptions):Promise putMany(blocks:AsyncIterable|Iterable, options?:AbortOptions):AsyncIterable } + +export type MessageProto = { + encode(value:any): Uint8Array + decode(bytes: Uint8Array): any + BlockPresenceType: { + Have: Have, + DontHave: DontHave + }, + Wantlist: { + WantType: { + Block: WantBlock + Have: HaveBlock + } + } +} diff --git a/src/types/message/message.proto.js b/src/types/message/message.proto.js index bb017d27..23748d25 100644 --- a/src/types/message/message.proto.js +++ b/src/types/message/message.proto.js @@ -2,6 +2,7 @@ const protons = require('protons') // from: https://github.com/ipfs/go-ipfs/blob/master/exchange/bitswap/message/pb/message.proto +/** @type {{Message: import('../../types').MessageProto}} */ module.exports = protons(` message Message { message Wantlist { diff --git a/tsconfig.json b/tsconfig.json index 4b69be3c..0cbdc31a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,42 +1,9 @@ { + "extends": "./node_modules/aegir/src/config/tsconfig.aegir.json", "compilerOptions": { - "allowJs": true, - "checkJs": true, - "forceConsistentCasingInFileNames": true, - "noImplicitReturns": false, - "noImplicitAny": false, - "noImplicitThis": true, - "noFallthroughCasesInSwitch": true, - "noUnusedLocals": true, - "noUnusedParameters": true, - "strictFunctionTypes": false, - "strictNullChecks": true, - "strictPropertyInitialization": true, - "strictBindCallApply": true, - "strict": true, - "alwaysStrict": true, - "esModuleInterop": true, - "target": "ES2018", - "moduleResolution": "node", - "declaration": true, - "declarationMap": true, - "outDir": "dist", - "skipLibCheck": true, - "stripInternal": true, - "resolveJsonModule": true, - "paths": { - "multiformats": [ - "src" - ] - }, - "baseUrl": "." + "outDir": "dist" }, "include": [ "src" - ], - "exclude": [ - "vendor", - "node_modules" - ], - "compileOnSave": false + ] } From 1c84c822a99c2bec6132249f636159ae4995d093 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Fri, 27 Nov 2020 12:50:45 -0800 Subject: [PATCH 04/13] fix: regression in typings --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5e3dd1f1..4548f070 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,7 @@ "it-length-prefixed": "^3.0.0", "it-pipe": "^1.1.0", "just-debounce-it": "^1.1.0", - "libp2p-interfaces": "^0.7.1", + "libp2p-interfaces": "git://github.com/libp2p/js-libp2p-interfaces.git#fix/types-of-multicodec-topology", "moving-average": "^1.0.0", "multicodec": "^2.0.0", "multihashing-async": "^2.0.1", From 1afa39dd912921276836545a8412a7fc9e5f7371 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Mon, 30 Nov 2020 21:05:45 -0800 Subject: [PATCH 05/13] chore: retrigger ci with changed deps From 49d0998dc6507f5d2611e6cec2aa86f42dde823e Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Mon, 30 Nov 2020 22:50:36 -0800 Subject: [PATCH 06/13] fix: add ts to dev-deps --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 4548f070..9989c606 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,8 @@ "sinon": "^9.0.0", "stats-lite": "^2.2.0", "uuid": "^8.0.0", - "@types/debug": "^4.1.5" + "@types/debug": "^4.1.5", + "typescript": "^4.0.5" }, "dependencies": { "abort-controller": "^3.0.0", From 815bcbd92e02215095e2d689e9362c08eb453f37 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 17 Dec 2020 18:20:01 +0000 Subject: [PATCH 07/13] chore: update deps, type tests as well as source --- package.json | 15 +- src/decision-engine/index.js | 4 +- src/decision-engine/ledger.js | 4 +- src/decision-engine/req-queue.js | 8 +- src/index.js | 16 +- src/network.js | 56 ++--- src/notifications.js | 4 +- src/stats/index.js | 10 +- src/types.ts | 5 - src/types/message/entry.js | 2 +- src/types/wantlist/entry.js | 2 +- src/types/wantlist/index.js | 2 +- src/utils/index.js | 2 +- src/want-manager/index.js | 2 +- src/want-manager/msg-queue.js | 2 +- test/benchmarks/get-many.js | 2 +- test/bitswap-mock-internals.js | 15 +- test/bitswap-stats.js | 4 +- test/bitswap.js | 2 +- test/decision-engine/decision-engine.js | 164 +++++++------- test/decision-engine/req-queue.spec.js | 272 +++++++++++++++++++++--- test/network/network.node.js | 20 +- test/notifications.spec.js | 10 +- test/swarms.js | 2 +- test/types/message.spec.js | 16 +- test/utils.spec.js | 23 +- test/utils/create-bitswap.js | 2 +- test/utils/create-temp-repo-browser.js | 6 +- test/utils/make-peer-id.js | 13 +- test/utils/mocks.js | 83 ++++++-- test/wantmanager/index.spec.js | 7 +- test/wantmanager/msg-queue.spec.js | 99 ++++----- tsconfig.json | 3 +- 33 files changed, 575 insertions(+), 302 deletions(-) diff --git a/package.json b/package.json index 9989c606..b792bf2a 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "src" ], "scripts": { + "prepare": "aegir build", "test": "aegir test", "test:browser": "aegir test -t browser -t webworker", "test:node": "aegir test -t node", @@ -34,7 +35,6 @@ "release-minor": "aegir release --type minor", "release-major": "aegir release --type major", "bench": "node benchmarks/index", - "build": "aegir build", "coverage": "aegir coverage --provider codecov", "docs": "aegir docs", "benchmarks": "node test/benchmarks/get-many" @@ -56,11 +56,12 @@ "homepage": "https://github.com/ipfs/js-ipfs-bitswap#readme", "devDependencies": { "@nodeutils/defaults-deep": "^1.1.0", - "aegir": "^29.1.0", + "@types/debug": "^4.1.5", + "aegir": "^29.2.0", "benchmark": "^2.1.4", "delay": "^4.3.0", "ipfs-repo": "^7.0.0", - "ipfs-utils": "^4.0.0", + "ipfs-utils": "^5.0.1", "iso-random-stream": "^1.1.1", "it-all": "^1.0.2", "it-drain": "^1.0.1", @@ -82,9 +83,8 @@ "rimraf": "^3.0.0", "sinon": "^9.0.0", "stats-lite": "^2.2.0", - "uuid": "^8.0.0", - "@types/debug": "^4.1.5", - "typescript": "^4.0.5" + "typescript": "^4.0.5", + "uuid": "^8.0.0" }, "dependencies": { "abort-controller": "^3.0.0", @@ -96,10 +96,11 @@ "it-length-prefixed": "^3.0.0", "it-pipe": "^1.1.0", "just-debounce-it": "^1.1.0", - "libp2p-interfaces": "git://github.com/libp2p/js-libp2p-interfaces.git#fix/types-of-multicodec-topology", + "libp2p-interfaces": "^0.8.1", "moving-average": "^1.0.0", "multicodec": "^2.0.0", "multihashing-async": "^2.0.1", + "native-abort-controller": "0.0.3", "protons": "^2.0.0", "streaming-iterables": "^5.0.2", "uint8arrays": "^1.1.0", diff --git a/src/decision-engine/index.js b/src/decision-engine/index.js index eaaa375f..c8363e0d 100644 --- a/src/decision-engine/index.js +++ b/src/decision-engine/index.js @@ -479,10 +479,10 @@ class DecisionEngine { module.exports = DecisionEngine /** - * @typedef {import('../types').PeerId} PeerId + * @typedef {import('peer-id')} PeerId * @typedef {import('../stats')} Stats * @typedef {import('../types').BlockData} BlockData - * @typedef {import('../types').Block} Block + * @typedef {import('ipld-block')} Block * @typedef {import('../types/message/entry')} BitswapMessageEntry * @typedef {import('../types/wantlist/entry')} WantListEntry * @typedef {import('../types').BlockStore} BlockStore diff --git a/src/decision-engine/ledger.js b/src/decision-engine/ledger.js index 0ef7ea0a..cec07cb9 100644 --- a/src/decision-engine/ledger.js +++ b/src/decision-engine/ledger.js @@ -76,8 +76,8 @@ class Ledger { module.exports = Ledger /** - * @typedef {import('../types').PeerId} PeerId - * @typedef {import('../types').CID} CID + * @typedef {import('peer-id')} PeerId + * @typedef {import('cids')} CID * @typedef {import('../types').WantType} WantType * @typedef {import('../types/wantlist/entry')} WantListEntry */ diff --git a/src/decision-engine/req-queue.js b/src/decision-engine/req-queue.js index f3b7512c..888a462c 100644 --- a/src/decision-engine/req-queue.js +++ b/src/decision-engine/req-queue.js @@ -26,10 +26,10 @@ const DefaultTaskMerger = { */ class RequestQueue { /** - * @param {TaskMerger} taskMerger + * @param {TaskMerger} [taskMerger] */ - constructor (taskMerger) { - this._taskMerger = taskMerger || DefaultTaskMerger + constructor (taskMerger = DefaultTaskMerger) { + this._taskMerger = taskMerger /** @type {SortedMap} */ this._byPeer = new SortedMap([], PeerTasks.compare) } @@ -420,7 +420,7 @@ module.exports = RequestQueue * @property {number} created * @property {Task} task * - * @typedef {import('../types').PeerId} PeerId + * @typedef {import('peer-id')} PeerId * @typedef {import('./interface').Task} Task * @typedef {import('./interface').TaskMerger} TaskMerger */ diff --git a/src/index.js b/src/index.js index f401deae..80c03a6a 100644 --- a/src/index.js +++ b/src/index.js @@ -6,7 +6,7 @@ const DecisionEngine = require('./decision-engine') const Notifications = require('./notifications') const logger = require('./utils').logger const Stats = require('./stats') -const AbortController = require('abort-controller').default +const AbortController = require('native-abort-controller') const { anySignal } = require('any-signal') const defaultOptions = { @@ -34,12 +34,12 @@ class Bitswap { /** * @param {LibP2P} libp2p * @param {BlockStore} blockstore - * @param {Object} options + * @param {Object} [options] * @param {boolean} [options.statsEnabled=false] * @param {number} [options.statsComputeThrottleTimeout=1000] * @param {number} [options.statsComputeThrottleMaxQueueSize=1000] */ - constructor (libp2p, blockstore, options) { + constructor (libp2p, blockstore, options = {}) { this._libp2p = libp2p this._log = logger(this.peerId) @@ -53,7 +53,7 @@ class Bitswap { }) // the network delivers messages - this.network = new Network(libp2p, this, {}, this._stats) + this.network = new Network(libp2p, this, this._stats) // local database this.blockstore = blockstore @@ -434,11 +434,11 @@ class Bitswap { module.exports = Bitswap /** - * @typedef {import('./types').LibP2P} LibP2P + * @typedef {import('libp2p')} LibP2P * @typedef {import('./types').BlockStore} BlockStore - * @typedef {import('./types').PeerId} PeerId + * @typedef {import('peer-id')} PeerId * @typedef {import('./types/message')} BitswapMessage - * @typedef {import('./types').Block} Block - * @typedef {import('./types').CID} CID + * @typedef {import('ipld-block')} Block + * @typedef {import('cids')} CID * @typedef {import('./types/wantlist/entry')} WantListEntry */ diff --git a/src/network.js b/src/network.js index 1f82aaba..db9bbbfa 100644 --- a/src/network.js +++ b/src/network.js @@ -17,20 +17,20 @@ class Network { /** * @param {LibP2P} libp2p * @param {BitSwap} bitswap - * @param {Object} options - * @param {boolean} [options.b100Only] * @param {Stats} stats + * @param {Object} [options] + * @param {boolean} [options.b100Only] */ - constructor (libp2p, bitswap, options, stats) { + constructor (libp2p, bitswap, stats, options = {}) { this._log = logger(libp2p.peerId, 'network') - options = options || {} - this.libp2p = libp2p - this.bitswap = bitswap - this.protocols = [BITSWAP100] + this._libp2p = libp2p + this._bitswap = bitswap + this._protocols = [BITSWAP100] + if (!options.b100Only) { // Latest bitswap first - this.protocols.unshift(BITSWAP110) - this.protocols.unshift(BITSWAP120) + this._protocols.unshift(BITSWAP110) + this._protocols.unshift(BITSWAP120) } this._stats = stats @@ -44,21 +44,21 @@ class Network { start () { this._running = true - this.libp2p.handle(this.protocols, this._onConnection) + this._libp2p.handle(this._protocols, this._onConnection) // register protocol with topology const topology = new MulticodecTopology({ - multicodecs: this.protocols, + multicodecs: this._protocols, handlers: { onConnect: this._onPeerConnect, onDisconnect: this._onPeerDisconnect } }) - this._registrarId = this.libp2p.registrar.register(topology) + this._registrarId = this._libp2p.registrar.register(topology) // All existing connections are like new ones for us - for (const peer of this.libp2p.peerStore.peers.values()) { - const conn = this.libp2p.connectionManager.get(peer.id) + for (const peer of this._libp2p.peerStore.peers.values()) { + const conn = this._libp2p.connectionManager.get(peer.id) conn && this._onPeerConnect(conn.remotePeer) } @@ -68,10 +68,10 @@ class Network { this._running = false // Unhandle both, libp2p doesn't care if it's not already handled - this.libp2p.unhandle(this.protocols) + this._libp2p.unhandle(this._protocols) // unregister protocol and handlers - this.libp2p.registrar.unregister(this._registrarId) + this._libp2p.registrar.unregister(this._registrarId) } /** @@ -96,9 +96,9 @@ class Network { for await (const data of source) { try { const message = await Message.deserialize(data.slice()) - await this.bitswap._receiveMessage(connection.remotePeer, message) + await this._bitswap._receiveMessage(connection.remotePeer, message) } catch (err) { - this.bitswap._receiveError(err) + this._bitswap._receiveError(err) break } } @@ -114,7 +114,7 @@ class Network { * @param {PeerId} peerId */ _onPeerConnect (peerId) { - this.bitswap._onPeerConnected(peerId) + this._bitswap._onPeerConnected(peerId) } /** @@ -123,7 +123,7 @@ class Network { * @returns {void} */ _onPeerDisconnect (peerId) { - this.bitswap._onPeerDisconnected(peerId) + this._bitswap._onPeerDisconnected(peerId) } /** @@ -136,7 +136,7 @@ class Network { * @returns {AsyncIterable} */ findProviders (cid, maxProviders, options = {}) { - return this.libp2p.contentRouting.findProviders( + return this._libp2p.contentRouting.findProviders( cid, { maxTimeout: CONSTANTS.providerRequestTimeout, @@ -172,7 +172,7 @@ class Network { * @returns {Promise} */ async provide (cid, options) { - await this.libp2p.contentRouting.provide(cid, options) + await this._libp2p.contentRouting.provide(cid, options) } /** @@ -224,7 +224,7 @@ class Network { throw new Error('network isn\'t running') } - return this.libp2p.dial(peer, options) + return this._libp2p.dial(peer, options) } /** @@ -234,7 +234,7 @@ class Network { * @param {PeerId|Multiaddr|Provider} peer */ _dialPeer (peer) { - return this.libp2p.dialProtocol(peer, [BITSWAP120, BITSWAP110, BITSWAP100]) + return this._libp2p.dialProtocol(peer, [BITSWAP120, BITSWAP110, BITSWAP100]) } /** @@ -273,10 +273,10 @@ async function writeMessage (stream, msg, log) { module.exports = Network /** - * @typedef {import('./types').PeerId} PeerId - * @typedef {import('./types').CID} CID - * @typedef {import('./types').Multiaddr} Multiaddr - * @typedef {import('./types').LibP2P} LibP2P + * @typedef {import('peer-id')} PeerId + * @typedef {import('cids')} CID + * @typedef {import('multiaddr')} Multiaddr + * @typedef {import('libp2p')} LibP2P * @typedef {import('./stats')} Stats * @typedef {import('./index')} BitSwap * diff --git a/src/notifications.js b/src/notifications.js index b6f98c4c..7b8f2463 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -115,6 +115,6 @@ class Notifications extends EventEmitter { module.exports = Notifications /** - * @typedef {import('./types').CID} CID - * @typedef {import('./types').PeerId} PeerId + * @typedef {import('cids')} CID + * @typedef {import('peer-id')} PeerId */ diff --git a/src/stats/index.js b/src/stats/index.js index 7e2531e4..c2a057f8 100644 --- a/src/stats/index.js +++ b/src/stats/index.js @@ -7,6 +7,9 @@ const Stat = require('./stat') * @typedef {[number, number, number]} AverageIntervals */ const defaultOptions = { + enabled: false, + computeThrottleTimeout: 1000, + computeThrottleMaxQueueSize: 1000, movingAverageIntervals: /** @type {AverageIntervals} */ ([ 60 * 1000, // 1 minute 5 * 60 * 1000, // 5 minutes @@ -16,14 +19,13 @@ const defaultOptions = { class Stats extends EventEmitter { /** - * - * @param {string[]} initialCounters + * @param {string[]} [initialCounters] * @param {Object} _options * @param {boolean} _options.enabled * @param {number} _options.computeThrottleTimeout * @param {number} _options.computeThrottleMaxQueueSize */ - constructor (initialCounters, _options) { + constructor (initialCounters = [], _options = defaultOptions) { super() const options = Object.assign({}, defaultOptions, _options) @@ -122,5 +124,5 @@ class Stats extends EventEmitter { module.exports = Stats /** - * @typedef {import('../types').PeerId} PeerId + * @typedef {import('peer-id')} PeerId */ diff --git a/src/types.ts b/src/types.ts index cc23f443..bc5a6474 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,11 +1,6 @@ import CID from 'cids' -import PeerId from 'peer-id' -import Multiaddr from 'multiaddr' -import LibP2P from 'libp2p' import Block from 'ipld-block' -export { CID, PeerId, Multiaddr, Block, LibP2P } - export type WantBlock = 0 export type HaveBlock = 1 export type WantType = WantBlock | HaveBlock diff --git a/src/types/message/entry.js b/src/types/message/entry.js index 2bcb0606..ca02d473 100644 --- a/src/types/message/entry.js +++ b/src/types/message/entry.js @@ -55,5 +55,5 @@ module.exports = class BitswapMessageEntry { /** * @typedef {import('../../types').WantType} WantType - * @typedef {import('../../types').CID} CID + * @typedef {import('cids')} CID */ diff --git a/src/types/wantlist/entry.js b/src/types/wantlist/entry.js index e65db309..09ac5539 100644 --- a/src/types/wantlist/entry.js +++ b/src/types/wantlist/entry.js @@ -49,5 +49,5 @@ module.exports = WantListEntry /** * @typedef {import('../../types').WantType} WantType - * @typedef {import('../../types').CID} CID + * @typedef {import('cids')} CID */ diff --git a/src/types/wantlist/index.js b/src/types/wantlist/index.js index 4a527e7c..e9cc9f21 100644 --- a/src/types/wantlist/index.js +++ b/src/types/wantlist/index.js @@ -113,7 +113,7 @@ Wantlist.Entry = Entry module.exports = Wantlist /** - * @typedef {import('../../types').CID} CID + * @typedef {import('cids')} CID * @typedef {import('../../types').WantType} WantType * @typedef {import('../../stats')} Stats */ diff --git a/src/utils/index.js b/src/utils/index.js index 138eb315..9aca6792 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -155,6 +155,6 @@ module.exports = { } /** - * @typedef {import('../types').PeerId} PeerId + * @typedef {import('peer-id')} PeerId * @typedef {import('../types/message')} BitswapMessageEntry */ diff --git a/src/want-manager/index.js b/src/want-manager/index.js index 9df65099..c27856a9 100644 --- a/src/want-manager/index.js +++ b/src/want-manager/index.js @@ -132,7 +132,7 @@ module.exports = class WantManager { } /** - * @typedef {import('../types').PeerId} PeerId + * @typedef {import('peer-id')} PeerId * @typedef {import('../network')} Network * @typedef {import('../stats')} Stats */ diff --git a/src/want-manager/msg-queue.js b/src/want-manager/msg-queue.js index 3a50412c..0b576740 100644 --- a/src/want-manager/msg-queue.js +++ b/src/want-manager/msg-queue.js @@ -71,6 +71,6 @@ module.exports = class MsgQueue { } /** - * @typedef {import('../types').PeerId} PeerId + * @typedef {import('peer-id')} PeerId * @typedef {import('../network')} Network */ diff --git a/test/benchmarks/get-many.js b/test/benchmarks/get-many.js index 46c7871f..0d063aad 100644 --- a/test/benchmarks/get-many.js +++ b/test/benchmarks/get-many.js @@ -4,7 +4,7 @@ const distributionTest = require('../utils/distribution-test') const print = require('./helpers/print-swarm-results') -const EventEmitter = require('events') +const { EventEmitter } = require('events') ;(async function () { const emitter = new EventEmitter() diff --git a/test/bitswap-mock-internals.js b/test/bitswap-mock-internals.js index 07f91b76..185e7780 100644 --- a/test/bitswap-mock-internals.js +++ b/test/bitswap-mock-internals.js @@ -11,7 +11,7 @@ const Message = require('../src/types/message') const Bitswap = require('../src') const CID = require('cids') const Block = require('ipld-block') -const AbortController = require('abort-controller') +const AbortController = require('native-abort-controller') const delay = require('delay') const createTempRepo = require('./utils/create-temp-repo-nodejs') @@ -20,11 +20,11 @@ const applyNetwork = require('./utils/mocks').applyNetwork const mockLibp2pNode = require('./utils/mocks').mockLibp2pNode const storeHasBlocks = require('./utils/store-has-blocks') const makeBlock = require('./utils/make-block') -const makePeerId = require('./utils/make-peer-id') +const { makePeerIds } = require('./utils/make-peer-id') const orderedFinish = require('./utils/helpers').orderedFinish function wantsBlock (cid, bitswap) { - for (const [key, value] of bitswap.getWantlist()) { // eslint-disable-line no-unused-vars + for (const [, value] of bitswap.getWantlist()) { if (value.cid.toString() === cid.toString()) { return true } @@ -43,7 +43,7 @@ describe('bitswap with mocks', function () { before(async () => { repo = await createTempRepo() blocks = await makeBlock(15) - ids = await makePeerId(2) + ids = await makePeerIds(2) }) after(() => repo.teardown()) @@ -112,7 +112,7 @@ describe('bitswap with mocks', function () { bs.start() - const others = await makePeerId(5) + const others = await makePeerIds(5) const blocks = await makeBlock(10) const messages = await Promise.all(range(5).map((i) => { @@ -182,6 +182,7 @@ describe('bitswap with mocks', function () { it('fails on requesting empty block', async () => { const bs = new Bitswap(mockLibp2pNode(), repo.blocks) try { + // @ts-expect-error we want this to fail await bs.get(null) } catch (err) { expect(err).to.exist() @@ -243,7 +244,7 @@ describe('bitswap with mocks', function () { setTimeout(() => { finish(1) - bs.put(block, () => {}) + bs.put(block) }, 200) const res = await get @@ -330,7 +331,7 @@ describe('bitswap with mocks', function () { const p1 = bs1.get(block.cid) setTimeout(() => { - bs2.put(block, () => {}) + bs2.put(block) }, 1000) const b1 = await p1 expect(b1).to.eql(block) diff --git a/test/bitswap-stats.js b/test/bitswap-stats.js index f61ea766..c9597cb9 100644 --- a/test/bitswap-stats.js +++ b/test/bitswap-stats.js @@ -9,7 +9,7 @@ const Bitswap = require('../src') const createTempRepo = require('./utils/create-temp-repo-nodejs') const createLibp2pNode = require('./utils/create-libp2p-node') const makeBlock = require('./utils/make-block') -const makePeerId = require('./utils/make-peer-id') +const { makePeerIds } = require('./utils/make-peer-id') const expectedStats = [ 'blocksReceived', @@ -39,7 +39,7 @@ describe('bitswap stats', () => { before(async () => { const nodes = [0, 1] blocks = await makeBlock(2) - ids = await makePeerId(2) + ids = await makePeerIds(2) // create 2 temp repos repos = await Promise.all(nodes.map(() => createTempRepo())) diff --git a/test/bitswap.js b/test/bitswap.js index 727c4097..8d8e5b08 100644 --- a/test/bitswap.js +++ b/test/bitswap.js @@ -83,7 +83,7 @@ describe('bitswap without DHT', function () { // incoming message with requested block from the other peer const message = new Message(false) - message.addEntry(block.cid, 1, false) + message.addEntry(block.cid, 1, Message.WantType.Block) message.addBlock(block) // slow blockstore diff --git a/test/decision-engine/decision-engine.js b/test/decision-engine/decision-engine.js index 04b7f8c6..12edc896 100644 --- a/test/decision-engine/decision-engine.js +++ b/test/decision-engine/decision-engine.js @@ -12,12 +12,14 @@ const multihashing = require('multihashing-async') const uint8ArrayFromString = require('uint8arrays/from-string') const uint8ArrayToString = require('uint8arrays/to-string') const drain = require('it-drain') +const defer = require('p-defer') const Message = require('../../src/types/message') const DecisionEngine = require('../../src/decision-engine') +const Stats = require('../../src/stats') const createTempRepo = require('../utils/create-temp-repo-nodejs.js') const makeBlock = require('../utils/make-block') -const makePeerId = require('../utils/make-peer-id') +const { makePeerId, makePeerIds } = require('../utils/make-peer-id') const mockNetwork = require('../utils/mocks').mockNetwork const sum = (nums) => nums.reduce((a, b) => a + b, 0) @@ -31,6 +33,10 @@ function stringifyMessages (messages) { return flatten(messages.map(messageToString)) } +/** + * + * @param {import('../../src/network')} network + */ async function newEngine (network) { const results = await Promise.all([ createTempRepo(), @@ -38,7 +44,7 @@ async function newEngine (network) { ]) const blockstore = results[0].blocks const peerId = results[1] - const engine = new DecisionEngine(peerId, blockstore, network || mockNetwork()) + const engine = new DecisionEngine(peerId, blockstore, network, new Stats()) engine.start() return { peer: peerId, engine: engine } } @@ -46,8 +52,8 @@ async function newEngine (network) { describe('Engine', () => { it('consistent accounting', async () => { const res = await Promise.all([ - newEngine(false), - newEngine(false) + newEngine(mockNetwork()), + newEngine(mockNetwork()) ]) const sender = res[0] @@ -79,8 +85,8 @@ describe('Engine', () => { it('peer is added to peers when message received or sent', async () => { const res = await Promise.all([ - newEngine(false), - newEngine(false) + newEngine(mockNetwork()), + newEngine(mockNetwork()) ]) const sanfrancisco = res[0] @@ -149,19 +155,15 @@ describe('Engine', () => { const set = testcase[0] const cancels = testcase[1] const keeps = difference(set, cancels) - - let network - const done = new Promise((resolve, reject) => { - network = mockNetwork(1, (res) => { - const msgs = stringifyMessages(res.messages) - expect(msgs.sort()).to.eql(keeps.sort()) - resolve() - }) + const deferred = defer() + const network = mockNetwork(1, (res) => { + const msgs = stringifyMessages(res.messages) + expect(msgs.sort()).to.eql(keeps.sort()) + deferred.resolve() }) - const id = await PeerId.create({ bits: 512 }) const repo = await createTempRepo() - const dEngine = new DecisionEngine(id, repo.blocks, network) + const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats()) dEngine.start() // Send wants then cancels for some of the wants @@ -171,14 +173,14 @@ describe('Engine', () => { // Simulate receiving blocks from the network await peerSendsBlocks(dEngine, repo, blocks, somePeer) - await done + await deferred.promise } } }) it('round-robins incoming wants', async () => { - const id = await makePeerId(1)[0] - const peers = await makePeerId(3) + const id = await makePeerId() + const peers = await makePeerIds(3) const blockSize = 256 * 1024 const blocks = await makeBlock(20, blockSize) @@ -194,76 +196,90 @@ describe('Engine', () => { const repo = await createTempRepo() await drain(repo.blocks.putMany(blocks)) - let network let rcvdBlockCount = 0 const received = new Map(peers.map(p => [p.toB58String(), { count: 0, bytes: 0 }])) - const done = new Promise((resolve) => { - network = mockNetwork(blocks.length, null, ([peer, msg]) => { - const pid = peer.toB58String() - const rcvd = received.get(pid) - - // Blocks should arrive in priority order. - // Note: we requested the blocks such that the priority order was - // highest at the start to lowest at the end. - for (const block of msg.blocks.values()) { - expect(blockIndex(block)).to.gte(rcvd.count) - } + const deferred = defer() + const network = mockNetwork(blocks.length, undefined, ([peer, msg]) => { + const pid = peer.toB58String() + const rcvd = received.get(pid) - rcvd.count += msg.blocks.size - rcvd.bytes += sum([...msg.blocks.values()].map(b => b.data.length)) + if (!rcvd) { + return deferred.reject(new Error(`Could not get received for peer ${pid}`)) + } + + // Blocks should arrive in priority order. + // Note: we requested the blocks such that the priority order was + // highest at the start to lowest at the end. + for (const block of msg.blocks.values()) { + expect(blockIndex(block)).to.gte(rcvd.count) + } - // pendingBytes should be equal to the remaining data we're expecting - expect(msg.pendingBytes).to.eql(blockSize * blocks.length - rcvd.bytes) + rcvd.count += msg.blocks.size + rcvd.bytes += sum([...msg.blocks.values()].map(b => b.data.length)) - // Expect each peer to receive blocks in a roughly round-robin fashion, - // in other words one peer shouldn't receive a bunch more blocks than - // the others at any given time. - for (const p of peers) { - if (p !== peer) { - const pCount = received.get(p.toB58String()).count - expect(rcvd.count - pCount).to.lt(blocks.length * 0.8) + // pendingBytes should be equal to the remaining data we're expecting + expect(msg.pendingBytes).to.eql(blockSize * blocks.length - rcvd.bytes) + + // Expect each peer to receive blocks in a roughly round-robin fashion, + // in other words one peer shouldn't receive a bunch more blocks than + // the others at any given time. + for (const p of peers) { + if (p !== peer) { + const peerCount = received.get(p.toB58String()) + + if (!peerCount) { + return deferred.reject(new Error(`Could not get peer count for ${p.toB58String()}`)) } + + const pCount = peerCount.count + expect(rcvd.count - pCount).to.lt(blocks.length * 0.8) } + } + + // When all peers have received all the blocks, we're done + rcvdBlockCount += msg.blocks.size + if (rcvdBlockCount === blocks.length * peers.length) { + // Make sure each peer received all blocks it was expecting + for (const peer of peers) { + const pid = peer.toB58String() + const rcvd = received.get(pid) - // When all peers have received all the blocks, we're done - rcvdBlockCount += msg.blocks.size - if (rcvdBlockCount === blocks.length * peers.length) { - // Make sure each peer received all blocks it was expecting - for (const peer of peers) { - const pid = peer.toB58String() - const rcvd = received.get(pid) - expect(rcvd.count).to.eql(blocks.length) + if (!rcvd) { + return deferred.reject(new Error(`Could not get peer count for ${pid}`)) } - resolve() + + expect(rcvd.count).to.eql(blocks.length) } - }) + + deferred.resolve() + } }) - const dEngine = new DecisionEngine(id, repo.blocks, network) + const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats()) dEngine.start() // Each peer requests all blocks for (const peer of peers) { const message = new Message(false) + for (const [i, block] of blocks.entries()) { message.addEntry(block.cid, blocks.length - i) } + await dEngine.messageReceived(peer, message) } - await done + await deferred.promise }) it('sends received blocks to peers that want them', async () => { - const [id, peer] = await makePeerId(2) + const [id, peer] = await makePeerIds(2) const blocks = await makeBlock(4, 8 * 1024) - let network - const receiveMessage = new Promise(resolve => { - network = mockNetwork(blocks.length, null, resolve) - }) + const deferred = defer() + const network = mockNetwork(blocks.length, undefined, ([peer, msg]) => deferred.resolve([peer, msg])) const repo = await createTempRepo() - const dEngine = new DecisionEngine(id, repo.blocks, network, null, { maxSizeReplaceHasWithBlock: 0 }) + const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) dEngine.start() const message = new Message(false) @@ -280,7 +296,7 @@ describe('Engine', () => { await dEngine.receivedBlocks(rcvdBlocks) // Wait till the engine sends a message - const [toPeer, msg] = await receiveMessage + const [toPeer, msg] = await deferred.promise // Expect the message to be sent to the peer that wanted the blocks expect(toPeer.toB58String()).to.eql(peer.toB58String()) @@ -294,18 +310,18 @@ describe('Engine', () => { }) it('sends DONT_HAVE', async () => { - const [id, peer] = await makePeerId(2) + const [id, peer] = await makePeerIds(2) const blocks = await makeBlock(4, 8 * 1024) let onMsg const receiveMessage = () => new Promise(resolve => { onMsg = resolve }) - const network = mockNetwork(blocks.length, null, (res) => { + const network = mockNetwork(blocks.length, undefined, (res) => { onMsg(res) }) const repo = await createTempRepo() - const dEngine = new DecisionEngine(id, repo.blocks, network, null, { maxSizeReplaceHasWithBlock: 0 }) + const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) dEngine.start() const message = new Message(false) @@ -344,7 +360,7 @@ describe('Engine', () => { }) it('handles want-have and want-block', async () => { - const [id, partner] = await makePeerId(2) + const [id, partner] = await makePeerIds(2) const alphabet = 'abcdefghijklmnopqrstuvwxyz' const vowels = 'aeiou' @@ -619,14 +635,14 @@ describe('Engine', () => { dEngine._processTasks() }) } - const network = mockNetwork(blocks.length, null, ([peer, msg]) => { + const network = mockNetwork(blocks.length, undefined, ([peer, msg]) => { onMsg && onMsg(msg) onMsg = undefined }) const repo = await createTempRepo() await drain(repo.blocks.putMany(blocks)) - const dEngine = new DecisionEngine(id, repo.blocks, network, null, { maxSizeReplaceHasWithBlock: 0 }) + const dEngine = new DecisionEngine(id, repo.blocks, network, new Stats(), { maxSizeReplaceHasWithBlock: 0 }) dEngine._scheduleProcessTasks = () => {} dEngine.start() @@ -698,7 +714,7 @@ describe('Engine', () => { // who is in the network const us = await newEngine(network) - const them = await newEngine() + const them = await newEngine(mockNetwork()) // add a block to our blockstore const data = uint8ArrayFromString(`this is message ${Date.now()}`) @@ -707,15 +723,11 @@ describe('Engine', () => { const block = new Block(data, cid) await us.engine.blockstore.put(block) + const message = new Message(false) + message.addEntry(cid, 1, Message.WantType.Block, false, false) + // receive a message with a want for our block - await us.engine.messageReceived(them.peer, { - blocks: [], - wantlist: [{ - cid, - priority: 1, - wantType: 'wanty' - }] - }) + await us.engine.messageReceived(them.peer, message) // should have added a task for the remote peer const tasks = us.engine._requestQueue._byPeer.get(them.peer.toB58String()) diff --git a/test/decision-engine/req-queue.spec.js b/test/decision-engine/req-queue.spec.js index 164dde3a..a335c5cd 100644 --- a/test/decision-engine/req-queue.spec.js +++ b/test/decision-engine/req-queue.spec.js @@ -32,15 +32,33 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 10, - priority: 3 + priority: 3, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'b', size: 5, - priority: 2 + priority: 2, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'c', size: 5, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) const { peerId, tasks, pendingSize } = rq.popTasks(11) @@ -59,11 +77,23 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 1, - priority: 2 + priority: 2, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'b', size: 1, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) const { tasks, pendingSize } = rq.popTasks(0) @@ -77,7 +107,13 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 1, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) const res = rq.popTasks(1) @@ -95,15 +131,33 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 1, - priority: 10 + priority: 10, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'b', size: 1, - priority: 5 + priority: 5, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'c', size: 1, - priority: 7 + priority: 7, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) const { peerId, tasks, pendingSize } = rq.popTasks(10) @@ -118,7 +172,13 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 1, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) // Pop all tasks for peer0 @@ -129,7 +189,13 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'b', size: 1, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) // Pop tasks for peer0 @@ -143,29 +209,65 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 5, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) rq.pushTasks(peerIds[1], [{ topic: 'b', size: 10, - priority: 3 + priority: 3, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'c', size: 3, - priority: 2 + priority: 2, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'd', size: 1, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) rq.pushTasks(peerIds[2], [{ topic: 'e', size: 7, - priority: 2 + priority: 2, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'f', size: 2, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) // Active Pending @@ -241,17 +343,35 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 0, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) rq.pushTasks(peerIds[1], [{ topic: 'a', size: 0, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) rq.pushTasks(peerIds[0], [{ topic: 'a', size: 1, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) // _byPeer map should have been resorted to put peer0 @@ -267,25 +387,55 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 1, - priority: 2 + priority: 2, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'b', size: 1, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) rq.pushTasks(peerIds[1], [{ topic: 'a', size: 1, - priority: 3 + priority: 3, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'b', size: 1, - priority: 2 + priority: 2, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'c', size: 1, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) rq.remove('a', peerIds[0]) @@ -308,7 +458,13 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 1, - priority: 2 + priority: 2, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) rq.remove('a', peerIds[1]) @@ -323,7 +479,13 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 1, - priority: 2 + priority: 2, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) rq.remove('b', peerIds[0]) @@ -340,17 +502,35 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 1, - priority: 2 + priority: 2, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'b', size: 1, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) rq.pushTasks(peerIds[0], [{ topic: 'b', size: 1, - priority: 3 + priority: 3, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) const { tasks } = rq.popTasks(10) @@ -363,21 +543,45 @@ describe('Request Queue', () => { rq.pushTasks(peerIds[0], [{ topic: 'a', size: 2, - priority: 2 + priority: 2, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'b', size: 1, - priority: 1 + priority: 1, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) rq.pushTasks(peerIds[1], [{ topic: 'c', size: 1, - priority: 3 + priority: 3, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }, { topic: 'd', size: 1, - priority: 2 + priority: 2, + data: { + blockSize: 0, + haveBlock: false, + isWantBlock: false, + sendDontHave: false + } }]) // Pop one task for each peer diff --git a/test/network/network.node.js b/test/network/network.node.js index 4a44830c..2b72b6b7 100644 --- a/test/network/network.node.js +++ b/test/network/network.node.js @@ -3,14 +3,19 @@ const { expect, assert } = require('aegir/utils/chai') const lp = require('it-length-prefixed') -const pipe = require('it-pipe') +const { pipe } = require('it-pipe') const pDefer = require('p-defer') const createLibp2pNode = require('../utils/create-libp2p-node') const makeBlock = require('../utils/make-block') const Network = require('../../src/network') const Message = require('../../src/types/message') +const Stats = require('../../src/stats') +/** + * @returns {import('../../src')} + */ function createBitswapMock () { + // @ts-ignore return { _receiveMessage: async () => {}, _receiveError: async () => {}, @@ -46,10 +51,10 @@ describe('network', () => { bitswapMockB = createBitswapMock() bitswapMockC = createBitswapMock() - networkA = new Network(p2pA, bitswapMockA) - networkB = new Network(p2pB, bitswapMockB) + networkA = new Network(p2pA, bitswapMockA, new Stats()) + networkB = new Network(p2pB, bitswapMockB, new Stats()) // only bitswap100 - networkC = new Network(p2pC, bitswapMockC, { b100Only: true }) + networkC = new Network(p2pC, bitswapMockC, new Stats(), { b100Only: true }) networkA.start() networkB.start() @@ -165,6 +170,7 @@ describe('network', () => { const ma = `${p2pB.multiaddrs[0]}/p2p/${p2pB.peerId.toB58String()}` const { stream } = await p2pA.dialProtocol(ma, '/ipfs/bitswap/' + version.num) + await pipe( [version.serialize(msg)], lp.encode(), @@ -238,11 +244,11 @@ describe('network', () => { }) it('dials to peer using Bitswap 1.2.0', async () => { - networkA = new Network(p2pA, bitswapMockA) + networkA = new Network(p2pA, bitswapMockA, new Stats()) // only supports 1.2.0 - networkB = new Network(p2pB, bitswapMockB) - networkB.protocols = ['/ipfs/bitswap/1.2.0'] + networkB = new Network(p2pB, bitswapMockB, new Stats()) + networkB._protocols = ['/ipfs/bitswap/1.2.0'] networkA.start() networkB.start() diff --git a/test/notifications.spec.js b/test/notifications.spec.js index 2f31826e..d2a8670e 100644 --- a/test/notifications.spec.js +++ b/test/notifications.spec.js @@ -4,13 +4,13 @@ const { expect } = require('aegir/utils/chai') const CID = require('cids') const Block = require('ipld-block') -const AbortController = require('abort-controller') +const AbortController = require('native-abort-controller') const uint8ArrayToString = require('uint8arrays/to-string') const Notifications = require('../src/notifications') const makeBlock = require('./utils/make-block') -const makePeerId = require('./utils/make-peer-id') +const { makePeerId } = require('./utils/make-peer-id') describe('Notifications', () => { let blocks @@ -54,7 +54,7 @@ describe('Notifications', () => { }) it('unwant block', async () => { - const n = new Notifications() + const n = new Notifications(peerId) const b = blocks[0] const p = n.wantBlock(b.cid) @@ -65,7 +65,7 @@ describe('Notifications', () => { }) it('abort block want', async () => { - const n = new Notifications() + const n = new Notifications(peerId) const b = blocks[0] const controller = new AbortController() @@ -103,7 +103,7 @@ describe('Notifications', () => { }) it('unwant block', async () => { - const n = new Notifications() + const n = new Notifications(peerId) const cid = new CID(blocks[0].cid.toV1().toString('base64')) const b = new Block(blocks[0].data, cid) diff --git a/test/swarms.js b/test/swarms.js index cef9f218..5cf93a5c 100644 --- a/test/swarms.js +++ b/test/swarms.js @@ -5,7 +5,7 @@ const stats = require('stats-lite') const distributionTest = require('./utils/distribution-test') -const EventEmitter = require('events') +const { EventEmitter } = require('events') const test = it describe.skip('swarms', () => { diff --git a/test/types/message.spec.js b/test/types/message.spec.js index d3a3f551..59eda5c8 100644 --- a/test/types/message.spec.js +++ b/test/types/message.spec.js @@ -46,10 +46,10 @@ describe('BitswapMessage', () => { msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, false, false) msg.addEntry(cids[0], 2, BitswapMessage.WantType.Have, true, false) - expect(msg.wantlist.get(cids[0].toString('base58btc')).priority).to.eql(1) + expect(msg.wantlist.get(cids[0].toString('base58btc'))).to.have.property('priority', 1) msg.addEntry(cids[0], 2, BitswapMessage.WantType.Block, true, false) - expect(msg.wantlist.get(cids[0].toString('base58btc')).priority).to.eql(2) + expect(msg.wantlist.get(cids[0].toString('base58btc'))).to.have.property('priority', 2) }) it('only changes from dont cancel to do cancel', () => { @@ -57,11 +57,11 @@ describe('BitswapMessage', () => { msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, true, false) msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, false, false) - expect(msg.wantlist.get(cids[0].toString('base58btc')).cancel).to.eql(true) + expect(msg.wantlist.get(cids[0].toString('base58btc'))).to.have.property('cancel', true) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Block, false, false) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Block, true, false) - expect(msg.wantlist.get(cids[1].toString('base58btc')).cancel).to.eql(true) + expect(msg.wantlist.get(cids[1].toString('base58btc'))).to.have.property('cancel', true) }) it('only changes from dont send to do send DONT_HAVE', () => { @@ -69,11 +69,11 @@ describe('BitswapMessage', () => { msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, false, false) msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, false, true) - expect(msg.wantlist.get(cids[0].toString('base58btc')).sendDontHave).to.eql(true) + expect(msg.wantlist.get(cids[0].toString('base58btc'))).to.have.property('sendDontHave', true) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Block, false, true) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Block, false, false) - expect(msg.wantlist.get(cids[1].toString('base58btc')).sendDontHave).to.eql(true) + expect(msg.wantlist.get(cids[1].toString('base58btc'))).to.have.property('sendDontHave', true) }) it('only override want-have with want-block (not vice versa)', () => { @@ -81,11 +81,11 @@ describe('BitswapMessage', () => { msg.addEntry(cids[0], 1, BitswapMessage.WantType.Block, false, false) msg.addEntry(cids[0], 1, BitswapMessage.WantType.Have, false, false) - expect(msg.wantlist.get(cids[0].toString('base58btc')).wantType).to.eql(BitswapMessage.WantType.Block) + expect(msg.wantlist.get(cids[0].toString('base58btc'))).to.have.property('wantType', BitswapMessage.WantType.Block) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Have, false, false) msg.addEntry(cids[1], 1, BitswapMessage.WantType.Block, false, false) - expect(msg.wantlist.get(cids[1].toString('base58btc')).wantType).to.eql(BitswapMessage.WantType.Block) + expect(msg.wantlist.get(cids[1].toString('base58btc'))).to.have.property('wantType', BitswapMessage.WantType.Block) }) }) diff --git a/test/utils.spec.js b/test/utils.spec.js index 46b8863a..f4c611ad 100644 --- a/test/utils.spec.js +++ b/test/utils.spec.js @@ -7,6 +7,7 @@ const Block = require('ipld-block') const multihashing = require('multihashing-async') const BitswapMessageEntry = require('../src/types/message/entry') const uint8ArrayFromString = require('uint8arrays/from-string') +const BitswapMessage = require('../src/types/message') const { groupBy, uniqWith, pullAllWith, includesWith, sortBy, isMapEqual } = require('../src/utils') const SortedMap = require('../src/utils/sorted-map') @@ -96,7 +97,7 @@ describe('utils spec', function () { } ] - const groupedList1 = sortBy(o => o.name, list) + const groupedList1 = sortBy(o => o.name.charCodeAt(0), list) const groupedList2 = sortBy(o => o.id, list) expect(groupedList1).to.be.deep.equal([{ id: 2, name: 'a' }, @@ -110,25 +111,25 @@ describe('utils spec', function () { describe('isMapEqual', () => { it('should on be false when !== size', () => { expect(isMapEqual( - new Map([['key1', 'value1'], ['key2', 'value2']]), - new Map([['key1', 'value1']]) + new Map([['key1', { data: uint8ArrayFromString('value1') }], ['key2', { data: uint8ArrayFromString('value2') }]]), + new Map([['key1', { data: uint8ArrayFromString('value1') }]]) )).to.be.false() }) it('should on be false if one key is missing', () => { expect(isMapEqual( - new Map([['key1', 'value1'], ['key2', 'value2']]), - new Map([['key1', 'value1'], ['key3', 'value2']]) + new Map([['key1', { data: uint8ArrayFromString('value1') }], ['key2', { data: uint8ArrayFromString('value2') }]]), + new Map([['key1', { data: uint8ArrayFromString('value1') }], ['key3', { data: uint8ArrayFromString('value2') }]]) )).to.be.false() }) - it('should on be false if BitswapMessageEntry dont match', async () => { + it('should on be false if BitswapMessageEntry don\'t match', async () => { const hash1 = await multihashing(uint8ArrayFromString('OMG!1'), 'sha2-256') const cid1 = new CID(1, 'dag-pb', hash1) expect(isMapEqual( - new Map([['key1', new BitswapMessageEntry(cid1, 1, true)], ['key2', new BitswapMessageEntry(cid1, 2, true)]]), - new Map([['key1', new BitswapMessageEntry(cid1, 1, true)], ['key2', new BitswapMessageEntry(cid1, 1, true)]]) + new Map([['key1', new BitswapMessageEntry(cid1, 1, BitswapMessage.WantType.Block)], ['key2', new BitswapMessageEntry(cid1, 2, BitswapMessage.WantType.Block)]]), + new Map([['key1', new BitswapMessageEntry(cid1, 1, BitswapMessage.WantType.Block)], ['key2', new BitswapMessageEntry(cid1, 1, BitswapMessage.WantType.Block)]]) )).to.be.false() }) @@ -137,8 +138,8 @@ describe('utils spec', function () { const cid1 = new CID(1, 'dag-pb', hash1) expect(isMapEqual( - new Map([['key1', new BitswapMessageEntry(cid1, 1, true)], ['key2', new BitswapMessageEntry(cid1, 1, true)]]), - new Map([['key1', new BitswapMessageEntry(cid1, 1, true)], ['key2', new BitswapMessageEntry(cid1, 1, true)]]) + new Map([['key1', new BitswapMessageEntry(cid1, 1, BitswapMessage.WantType.Block)], ['key2', new BitswapMessageEntry(cid1, 1, BitswapMessage.WantType.Block)]]), + new Map([['key1', new BitswapMessageEntry(cid1, 1, BitswapMessage.WantType.Block)], ['key2', new BitswapMessageEntry(cid1, 1, BitswapMessage.WantType.Block)]]) )).to.be.true() }) @@ -212,7 +213,7 @@ describe('utils spec', function () { expect(sm.get('two')).to.eql(2) - sm.clear('two') + sm.clear() expect(sm.get('two')).to.be.undefined() expect(sm.size).to.eql(0) diff --git a/test/utils/create-bitswap.js b/test/utils/create-bitswap.js index fd3dc32d..80edfbd1 100644 --- a/test/utils/create-bitswap.js +++ b/test/utils/create-bitswap.js @@ -1,6 +1,6 @@ 'use strict' -const Bitswap = require('../..') +const Bitswap = require('../../src') const createTempRepo = require('./create-temp-repo-nodejs') const createLibp2pNode = require('./create-libp2p-node') diff --git a/test/utils/create-temp-repo-browser.js b/test/utils/create-temp-repo-browser.js index c06a7725..c7b23c76 100644 --- a/test/utils/create-temp-repo-browser.js +++ b/test/utils/create-temp-repo-browser.js @@ -3,10 +3,8 @@ const IPFSRepo = require('ipfs-repo') -const idb = self.indexedDB || - self.mozIndexedDB || - self.webkitIndexedDB || - self.msIndexedDB +// @ts-ignore +const idb = self.indexedDB || self.mozIndexedDB || self.webkitIndexedDB || self.msIndexedDB async function createTempRepo () { const date = Date.now().toString() diff --git a/test/utils/make-peer-id.js b/test/utils/make-peer-id.js index 87d3b5ab..3d74c33c 100644 --- a/test/utils/make-peer-id.js +++ b/test/utils/make-peer-id.js @@ -2,9 +2,18 @@ const PeerId = require('peer-id') -module.exports = async (count) => { +async function makePeerId () { + return (await makePeerIds(1))[0] +} + +async function makePeerIds (count) { const peerIds = await Promise.all([...new Array(count || 1)].map(() => { return PeerId.create({ bits: 512 }) })) - return count ? peerIds : peerIds[0] + return peerIds +} + +module.exports = { + makePeerId, + makePeerIds } diff --git a/test/utils/mocks.js b/test/utils/mocks.js index 7dab93de..3290c54a 100644 --- a/test/utils/mocks.js +++ b/test/utils/mocks.js @@ -8,9 +8,43 @@ const PeerStore = require('libp2p/src/peer-store') const Node = require('./create-libp2p-node').bundle const tmpdir = require('ipfs-utils/src/temp-dir') const Repo = require('ipfs-repo') -const EventEmitter = require('events') +const { EventEmitter } = require('events') +const toString = require('uint8arrays/to-string') const Bitswap = require('../../src') +const Network = require('../../src/network') +const Stats = require('../../src/stats') + +/** + * @typedef {import('../../src/types').BlockStore} BlockStore + */ + +/** + * + * @returns {BlockStore} + */ +function mockBlockStore () { + const blocks = {} + + const store = { + has: (cid) => Promise.resolve(Boolean(blocks[toString(cid.multihash)])), + get: (cid) => Promise.resolve(blocks[toString(cid.multihash)]), + put: (block) => { + blocks[toString(block.cid.multihash)] = block + + return Promise.resolve(block) + }, + putMany: async function * (blocks) { + for await (const block of blocks) { + store.put(block) + + yield block + } + } + } + + return store +} /* * Create a mock libp2p node @@ -47,12 +81,15 @@ exports.mockLibp2pNode = () => { }) } -/* +/** * Create a mock network instance + * + * @param {number} [calls] + * @param {Function} [done] + * @param {Function} [onMsg] + * @returns {import('../../src/network')} */ -exports.mockNetwork = (calls, done, onMsg) => { - done = done || (() => {}) - +exports.mockNetwork = (calls = Infinity, done = () => {}, onMsg = () => {}) => { const connects = [] const messages = [] let i = 0 @@ -60,18 +97,27 @@ exports.mockNetwork = (calls, done, onMsg) => { const finish = (msgTo) => { onMsg && onMsg(msgTo) if (++i === calls) { - done({ connects: connects, messages: messages }) + done && done({ connects: connects, messages: messages }) } } - return { - messages, - connects, + class MockNetwork extends Network { + constructor () { + super({}, new Bitswap({}, mockBlockStore()), new Stats()) + + this.connects = connects + this.messages = messages + } + + // @ts-ignore connectTo (p) { setTimeout(() => { connects.push(p) }) - }, + + return Promise.resolve({ id: '', remotePeer: '' }) + } + sendMessage (p, msg) { messages.push([p, msg]) @@ -80,20 +126,27 @@ exports.mockNetwork = (calls, done, onMsg) => { }) return Promise.resolve() - }, + } + start () { return Promise.resolve() - }, + } + stop () { return Promise.resolve() - }, + } + findAndConnect () { return Promise.resolve() - }, + } + provide () { return Promise.resolve() } } + + // @ts-ignore + return new MockNetwork() } /* @@ -115,7 +168,7 @@ exports.createMockTestNet = async (repo, count) => { connectTo (id) { return new Promise((resolve, reject) => { if (!hexIds.includes(hexIds, id.toHexString())) { - return reject(new Error('unkown peer')) + return reject(new Error('unknown peer')) } resolve() }) diff --git a/test/wantmanager/index.spec.js b/test/wantmanager/index.spec.js index d4d1f782..6adb3f51 100644 --- a/test/wantmanager/index.spec.js +++ b/test/wantmanager/index.spec.js @@ -6,16 +6,17 @@ const { expect } = require('aegir/utils/chai') const cs = require('../../src/constants') const Message = require('../../src/types/message') const WantManager = require('../../src/want-manager') +const Stats = require('../../src/stats') const mockNetwork = require('../utils/mocks').mockNetwork const makeBlock = require('../utils/make-block') -const makePeerId = require('../utils/make-peer-id') +const { makePeerIds } = require('../utils/make-peer-id') describe('WantManager', () => { it('sends wantlist to all connected peers', async function () { this.timeout(80 * 1000) - const peerIds = await makePeerId(3) + const peerIds = await makePeerIds(3) const blocks = await makeBlock(3) const cids = blocks.map((b) => b.cid) @@ -56,7 +57,7 @@ describe('WantManager', () => { resolve() }) - const wantManager = new WantManager(peerIds[2], network) + const wantManager = new WantManager(peerIds[2], network, new Stats()) wantManager.start() wantManager.wantBlocks([cid1, cid2]) diff --git a/test/wantmanager/msg-queue.spec.js b/test/wantmanager/msg-queue.spec.js index fda8cf24..1860189d 100644 --- a/test/wantmanager/msg-queue.spec.js +++ b/test/wantmanager/msg-queue.spec.js @@ -7,6 +7,10 @@ const CID = require('cids') const multihashing = require('multihashing-async') const Message = require('../../src/types/message') const MsgQueue = require('../../src/want-manager/msg-queue') +const defer = require('p-defer') +const { + mockNetwork +} = require('../utils/mocks') describe('MessageQueue', () => { let peerIds @@ -20,7 +24,7 @@ describe('MessageQueue', () => { cids = hashes.map((h) => new CID(h)) }) - it('connects and sends messages', () => { + it('connects and sends messages', async () => { const msg = new Message(true) const cid1 = cids[0] const cid2 = cids[1] @@ -32,60 +36,45 @@ describe('MessageQueue', () => { msg.addEntry(cid1, 3) msg.addEntry(cid2, 1) - const messages = [] - const connects = [] - let i = 0 - - return new Promise((resolve) => { - const finish = () => { - i++ - if (i === 2) { - expect(connects).to.be.eql([peerIds[1], peerIds[1]]) - - const m1 = new Message(false) - m1.addEntry(cid3, 1) - m1.addEntry(cid4, 2) - m1.cancel(cid5) - m1.cancel(cid6) - - expect( - messages - ).to.be.eql([ - [peerIds[1], msg], - [peerIds[1], m1] - ]) - - resolve() - } - } - - const network = { - async connectTo (p) { // eslint-disable-line require-await - connects.push(p) - }, - async sendMessage (p, msg) { // eslint-disable-line require-await - messages.push([p, msg]) - finish() - } - } - - const mq = new MsgQueue(peerIds[0], peerIds[1], network) - - expect(mq.refcnt).to.equal(1) - - const batch1 = [ - new Message.Entry(cid3, 1, Message.WantType.Block, false), - new Message.Entry(cid4, 2, Message.WantType.Block, false) - ] - - const batch2 = [ - new Message.Entry(cid5, 1, Message.WantType.Block, true), - new Message.Entry(cid6, 2, Message.WantType.Block, true) - ] - - mq.addEntries(batch1) - mq.addEntries(batch2) - mq.addMessage(msg) + const deferred = defer() + + const network = mockNetwork(2, ({ connects, messages }) => { + expect(connects).to.be.eql([peerIds[1], peerIds[1]]) + + const m1 = new Message(false) + m1.addEntry(cid3, 1) + m1.addEntry(cid4, 2) + m1.cancel(cid5) + m1.cancel(cid6) + + expect( + messages + ).to.be.eql([ + [peerIds[1], msg], + [peerIds[1], m1] + ]) + + deferred.resolve() }) + + const mq = new MsgQueue(peerIds[0], peerIds[1], network) + + expect(mq.refcnt).to.equal(1) + + const batch1 = [ + new Message.Entry(cid3, 1, Message.WantType.Block, false), + new Message.Entry(cid4, 2, Message.WantType.Block, false) + ] + + const batch2 = [ + new Message.Entry(cid5, 1, Message.WantType.Block, true), + new Message.Entry(cid6, 2, Message.WantType.Block, true) + ] + + mq.addEntries(batch1) + mq.addEntries(batch2) + mq.addMessage(msg) + + await deferred.promise }) }) diff --git a/tsconfig.json b/tsconfig.json index 0cbdc31a..77830df2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,6 +4,7 @@ "outDir": "dist" }, "include": [ - "src" + "src", + "test" ] } From bd9d4db29aa4a03ee300339721df3d4897a606f6 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 17 Dec 2020 18:25:49 +0000 Subject: [PATCH 08/13] chore: update typecheck version --- .github/workflows/typecheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index 624310c2..61463746 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -24,4 +24,4 @@ jobs: - name: Install dependencies run: npm install - name: Typecheck - uses: gozala/typescript-error-reporter-action@v1.0.4 + uses: gozala/typescript-error-reporter-action@v1.0.8 From e51719e8b9c3ea940e33b27e1fbe894f89d9596b Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Thu, 4 Mar 2021 13:45:11 -0800 Subject: [PATCH 09/13] chore: update to new aegir --- .aegir.js | 46 ++++++----- .github/workflows/main.yml | 78 +++++++++++++++++++ .travis.yml | 59 -------------- package.json | 69 +++++++++------- scripts/node-globals.js | 4 + src/decision-engine/index.js | 55 +++++++++---- src/decision-engine/ledger.js | 17 ++-- src/decision-engine/req-queue.js | 34 ++++---- src/decision-engine/task-merger.js | 9 ++- .../{interface.ts => types.d.ts} | 4 +- src/index.js | 39 ++++++---- src/network.js | 59 +++++++------- src/notifications.js | 12 ++- src/stats/index.js | 15 +++- src/stats/stat.js | 9 ++- src/types.ts | 74 ------------------ src/types/message/entry.js | 12 ++- src/types/message/index.js | 43 +++++----- src/types/message/message.proto.d.ts | 73 +++++++++++++++++ src/types/message/message.proto.js | 3 +- src/types/wantlist/entry.js | 20 +++-- src/types/wantlist/index.js | 16 ++-- src/utils/index.js | 9 +-- src/utils/sorted-map.js | 8 +- src/want-manager/index.js | 67 ++++++++++++---- src/want-manager/msg-queue.js | 30 +++++-- test/bitswap-mock-internals.js | 2 +- test/notifications.spec.js | 2 +- test/types/message.spec.js | 2 +- test/utils/connect-all.js | 5 ++ test/utils/create-libp2p-node.js | 8 +- test/utils/create-temp-repo-browser.js | 2 + test/utils/create-temp-repo-nodejs.js | 8 +- test/utils/distribution-test.js | 9 +++ test/utils/helpers.js | 8 ++ test/utils/make-block.js | 8 ++ test/utils/make-peer-id.js | 4 + test/wantmanager/msg-queue.spec.js | 2 + tsconfig.json | 3 +- 39 files changed, 559 insertions(+), 368 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 .travis.yml create mode 100644 scripts/node-globals.js rename src/decision-engine/{interface.ts => types.d.ts} (91%) delete mode 100644 src/types.ts create mode 100644 src/types/message/message.proto.d.ts diff --git a/.aegir.js b/.aegir.js index d096ba1a..74902375 100644 --- a/.aegir.js +++ b/.aegir.js @@ -1,25 +1,33 @@ 'use strict' +const path = require('path') -module.exports = { - bundlesize: { maxSize: '68kB' }, - karma: { - files: [{ - pattern: 'test/test-data/**/*', - watched: false, - served: true, - included: false - }] - }, - webpack: { - node: { - // needed by ipfs-repo-migrations - path: true, - - // needed by dependencies of peer-id - stream: true, +/** @type {import('aegir').Options["build"]["config"]} */ +const esbuild = { + // this will inject all the named exports from 'node-globals.js' as globals + inject: [require.resolve('./scripts/node-globals.js')], + plugins: [ + { + name: 'node built ins', // this will make the bundler resolve node builtins to the respective browser polyfill + setup (build) { + build.onResolve({ filter: /^stream$/ }, () => { + return { path: require.resolve('readable-stream') } + }) + } + } + ] +} - // needed by core-util-is - Buffer: true +/** @type {import('aegir').PartialOptions} */ +module.exports = { + test: { + browser :{ + config: { + buildConfig: esbuild + } } + }, + build: { + bundlesize: { maxSize: '68kB' }, + config: esbuild } } diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..1cfde4ec --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,78 @@ +name: ci +on: + push: + branches: + - master + - 'release/**' + pull_request: + branches: + - master + - 'release/**' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: npm install + - run: npx aegir lint + - run: npx aegir build + - run: npx aegir dep-check + - uses: ipfs/aegir/actions/bundle-size@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + test-node: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + node: [14, 15] + fail-fast: true + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node }} + - run: npm install + - run: npx aegir test -t node --bail --cov + - uses: codecov/codecov-action@v1 + test-chrome: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: microsoft/playwright-github-action@v1 + - run: npm install + - run: npx aegir test -t browser -t webworker --bail # add --cov later when its fixed + - uses: codecov/codecov-action@v1 + test-firefox: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: microsoft/playwright-github-action@v1 + - run: npm install + - run: npx aegir test -t browser -t webworker --bail -- --browser firefox + test-webkit: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: microsoft/playwright-github-action@v1 + - run: npm install + - run: npx aegir test -t browser -t webworker --bail --timeout 10000 -- --browser webkit + test-electron-main: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: npm install + - run: npx xvfb-maybe aegir test -t electron-main --bail + test-electron-renderer: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: npm install + - run: npx xvfb-maybe aegir test -t electron-renderer --bail diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2398cfd5..00000000 --- a/.travis.yml +++ /dev/null @@ -1,59 +0,0 @@ -language: node_js -cache: npm - -branches: - only: - - master - - /^release\/.*$/ - -node_js: - - 'lts/*' - - 'node' - -stages: - - check - - test - - cov - -os: - - linux - - osx - - windows - -script: npx nyc -s npm run test:node -- --bail -after_success: npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov - -jobs: - include: - - stage: check - script: - - npx aegir build --bundlesize - - npx aegir dep-check - - npm run lint - - - stage: test - name: chrome - addons: - chrome: stable - script: npx aegir test -t browser -t webworker - - - stage: test - name: firefox - addons: - firefox: latest - script: npx aegir test -t browser -t webworker -- --browsers FirefoxHeadless - - - stage: test - name: electron-main - os: osx - script: - - npx aegir test -t electron-main --bail - - - stage: test - name: electron-renderer - os: osx - script: - - npx aegir test -t electron-renderer --bail - -notifications: - email: false diff --git a/package.json b/package.json index b792bf2a..252ba910 100644 --- a/package.json +++ b/package.json @@ -14,18 +14,22 @@ "src/*": [ "dist/src/*", "dist/src/*/index" + ], + "src/": [ + "dist/src/index" ] } }, "eslintConfig": { - "extends": "ipfs" + "extends": "ipfs", + "ignorePatterns": ["scripts/*"] }, "files": [ "dist", "src" ], "scripts": { - "prepare": "aegir build", + "prepare": "aegir build --no-bundle", "test": "aegir test", "test:browser": "aegir test -t browser -t webworker", "test:node": "aegir test -t node", @@ -35,7 +39,7 @@ "release-minor": "aegir release --type minor", "release-major": "aegir release --type major", "bench": "node benchmarks/index", - "coverage": "aegir coverage --provider codecov", + "coverage": "aegir test -t node --cov && nyc report --reporter=html", "docs": "aegir docs", "benchmarks": "node test/benchmarks/get-many" }, @@ -57,53 +61,58 @@ "devDependencies": { "@nodeutils/defaults-deep": "^1.1.0", "@types/debug": "^4.1.5", - "aegir": "^29.2.0", + "aegir": "^31.0.4", + "assert": "^2.0.0", "benchmark": "^2.1.4", - "delay": "^4.3.0", + "delay": "^5.0.0", "ipfs-repo": "^7.0.0", - "ipfs-utils": "^5.0.1", + "ipfs-utils": "^6.0.1", "iso-random-stream": "^1.1.1", - "it-all": "^1.0.2", - "it-drain": "^1.0.1", - "libp2p": "^0.29.3", - "libp2p-kad-dht": "^0.20.0", - "libp2p-mplex": "^0.10.0", - "libp2p-secio": "^0.13.0", - "libp2p-tcp": "^0.15.0", + "it-all": "^1.0.5", + "it-drain": "^1.0.4", + "libp2p": "^0.30.9", + "libp2p-kad-dht": "^0.21.0", + "libp2p-mplex": "^0.10.2", + "libp2p-secio": "^0.13.1", + "libp2p-tcp": "^0.15.3", "lodash.difference": "^4.5.0", "lodash.flatten": "^4.4.0", "lodash.range": "^3.2.0", "lodash.without": "^4.4.0", "ncp": "^2.0.0", "p-defer": "^3.0.0", - "p-event": "^4.1.0", - "p-wait-for": "^3.1.0", - "peer-id": "^0.14.0", + "p-event": "^4.2.0", + "p-wait-for": "^3.2.0", + "peer-id": "^0.14.3", "promisify-es6": "^1.0.3", - "rimraf": "^3.0.0", - "sinon": "^9.0.0", + "rimraf": "^3.0.2", + "sinon": "^9.2.4", "stats-lite": "^2.2.0", - "typescript": "^4.0.5", - "uuid": "^8.0.0" + "uuid": "^8.3.2" }, "dependencies": { "abort-controller": "^3.0.0", - "any-signal": "^2.1.1", + "any-signal": "^2.1.2", "bignumber.js": "^9.0.0", - "cids": "^1.0.0", + "cids": "^1.1.6", "debug": "^4.2.0", + "ipfs-core-types": "^0.3.0", "ipld-block": "^0.11.0", - "it-length-prefixed": "^3.0.0", + "it-length-prefixed": "^3.1.0", "it-pipe": "^1.1.0", "just-debounce-it": "^1.1.0", - "libp2p-interfaces": "^0.8.1", - "moving-average": "^1.0.0", - "multicodec": "^2.0.0", - "multihashing-async": "^2.0.1", - "native-abort-controller": "0.0.3", + "libp2p-interfaces": "^0.8.3", + "moving-average": "git://github.com/gozala/moving-average#types", + "multicodec": "^3.0.1", + "multihashing-async": "^2.1.2", + "native-abort-controller": "^1.0.3", + "process": "^0.11.10", "protons": "^2.0.0", - "streaming-iterables": "^5.0.2", - "uint8arrays": "^1.1.0", + "readable-stream": "^3.6.0", + "streaming-iterables": "^5.0.4", + "uint8arrays": "^2.1.3", + "url": "^0.11.0", + "util": "^0.12.3", "varint-decoder": "^1.0.0" }, "pre-push": [ diff --git a/scripts/node-globals.js b/scripts/node-globals.js new file mode 100644 index 00000000..f5912dfc --- /dev/null +++ b/scripts/node-globals.js @@ -0,0 +1,4 @@ +import { Buffer } from 'buffer' +import process from "process/browser" + +export { Buffer, process } diff --git a/src/decision-engine/index.js b/src/decision-engine/index.js index c8363e0d..f2395cdd 100644 --- a/src/decision-engine/index.js +++ b/src/decision-engine/index.js @@ -1,5 +1,11 @@ 'use strict' +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('ipfs-core-types/src/block-service').Block} Block + * @typedef {import('../types/message/entry')} BitswapMessageEntry + */ + const CID = require('cids') const Message = require('../types/message') @@ -27,16 +33,15 @@ const MAX_SIZE_REPLACE_HAS_WITH_BLOCK = 1024 class DecisionEngine { /** - * * @param {PeerId} peerId - * @param {BlockStore} blockstore + * @param {import('ipfs-core-types/src/block-store').BlockStore} blockstore * @param {import('../network')} network - * @param {Stats} stats + * @param {import('../stats')} stats * @param {Object} [opts] * @param {number} [opts.targetMessageSize] * @param {number} [opts.maxSizeReplaceHasWithBlock] */ - constructor (peerId, blockstore, network, stats, opts) { + constructor (peerId, blockstore, network, stats, opts = {}) { this._log = logger(peerId, 'engine') this.blockstore = blockstore this.network = network @@ -52,6 +57,12 @@ class DecisionEngine { this._requestQueue = new RequestQueue(TaskMerger) } + /** + * @template {Object} Opts + * @param {Opts} opts + * @returns {Opts & {maxSizeReplaceHasWithBlock:number, targetMessageSize:number}} + * @private + */ _processOpts (opts) { return { maxSizeReplaceHasWithBlock: MAX_SIZE_REPLACE_HAS_WITH_BLOCK, @@ -60,14 +71,21 @@ class DecisionEngine { } } + /** + * @private + */ _scheduleProcessTasks () { setTimeout(() => { this._processTasks() }) } - // Pull tasks off the request queue and send a message to the corresponding - // peer + /** + * Pull tasks off the request queue and send a message to the corresponding + * peer + * + * @private + */ async _processTasks () { if (!this._running) { return @@ -152,7 +170,7 @@ class DecisionEngine { /** * @param {PeerId} peerId - * @returns {Map} + * @returns {Map} */ wantlistForPeer (peerId) { const peerIdStr = peerId.toB58String() @@ -162,6 +180,7 @@ class DecisionEngine { /** * @param {PeerId} peerId + * @returns {import('ipfs-core-types/src/bitswap').LedgerForPeer|null} */ ledgerForPeer (peerId) { const peerIdStr = peerId.toB58String() @@ -262,7 +281,9 @@ class DecisionEngine { } // Clear cancelled wants and add new wants to the ledger + /** @type {CID[]} */ const cancels = [] + /** @type {BitswapMessageEntry[]} */ const wants = [] msg.wantlist.forEach((entry) => { if (entry.cancel) { @@ -356,6 +377,11 @@ class DecisionEngine { } } + /** + * @private + * @param {import('../types/message/message.proto').WantType} wantType + * @param {number} blockSize + */ _sendAsBlock (wantType, blockSize) { return wantType === WantType.Block || blockSize <= this._opts.maxSizeReplaceHasWithBlock @@ -391,6 +417,11 @@ class DecisionEngine { return res } + /** + * @private + * @param {Map} blocksMap + * @param {Ledger} ledger + */ _updateBlockAccounting (blocksMap, ledger) { blocksMap.forEach(b => { this._log('got block (%s bytes)', b.data.length) @@ -477,13 +508,3 @@ class DecisionEngine { } module.exports = DecisionEngine - -/** - * @typedef {import('peer-id')} PeerId - * @typedef {import('../stats')} Stats - * @typedef {import('../types').BlockData} BlockData - * @typedef {import('ipld-block')} Block - * @typedef {import('../types/message/entry')} BitswapMessageEntry - * @typedef {import('../types/wantlist/entry')} WantListEntry - * @typedef {import('../types').BlockStore} BlockStore - */ diff --git a/src/decision-engine/ledger.js b/src/decision-engine/ledger.js index cec07cb9..48f4fe3e 100644 --- a/src/decision-engine/ledger.js +++ b/src/decision-engine/ledger.js @@ -2,9 +2,13 @@ const Wantlist = require('../types/wantlist') +/** + * @typedef {import('cids')} CID + */ + class Ledger { /** - * @param {PeerId} peerId + * @param {import('peer-id')} peerId */ constructor (peerId) { this.partner = peerId @@ -41,7 +45,7 @@ class Ledger { * * @param {CID} cid * @param {number} priority - * @param {WantType} [wantType] + * @param {import('../types/message/message.proto').WantType} wantType * @returns {void} */ wants (cid, priority, wantType) { @@ -59,7 +63,7 @@ class Ledger { /** * @param {CID} cid - * @returns {WantListEntry|void} + * @returns {import('../types/wantlist/entry')|void} */ wantlistContains (cid) { return this.wantlist.contains(cid) @@ -74,10 +78,3 @@ class Ledger { } module.exports = Ledger - -/** - * @typedef {import('peer-id')} PeerId - * @typedef {import('cids')} CID - * @typedef {import('../types').WantType} WantType - * @typedef {import('../types/wantlist/entry')} WantListEntry - */ diff --git a/src/decision-engine/req-queue.js b/src/decision-engine/req-queue.js index 888a462c..cbe94050 100644 --- a/src/decision-engine/req-queue.js +++ b/src/decision-engine/req-queue.js @@ -2,6 +2,21 @@ const SortedMap = require('../utils/sorted-map') +/** + * @typedef {Object} PopTaskResult + * @property {PeerId} [peerId] + * @property {Task[]} tasks + * @property {number} pendingSize + * + * @typedef {Object} PendingTask + * @property {number} created + * @property {Task} task + * + * @typedef {import('peer-id')} PeerId + * @typedef {import('./types').Task} Task + * @typedef {import('./types').TaskMerger} TaskMerger + */ + /** * The task merger that is used by default. * Assumes that new tasks do not add any information over existing tasks, @@ -100,9 +115,11 @@ class RequestQueue { return undefined } + // eslint-disable-next-line no-unreachable-loop for (const [, v] of this._byPeer) { return v } + return undefined } @@ -233,7 +250,7 @@ class PeerTasks { * Pop tasks off the queue such that the total size is at least targetMinBytes * * @param {number} targetMinBytes - * @returns {Object} + * @returns {PopTaskResult} */ popTasks (targetMinBytes) { let size = 0 @@ -409,18 +426,3 @@ class PendingTasks { } module.exports = RequestQueue - -/** - * @typedef {Object} PopTaskResult - * @property {PeerId} [peerId] - * @property {Task[]} tasks - * @property {number} pendingSize - * - * @typedef {Object} PendingTask - * @property {number} created - * @property {Task} task - * - * @typedef {import('peer-id')} PeerId - * @typedef {import('./interface').Task} Task - * @typedef {import('./interface').TaskMerger} TaskMerger - */ diff --git a/src/decision-engine/task-merger.js b/src/decision-engine/task-merger.js index 2f16aacc..18941f52 100644 --- a/src/decision-engine/task-merger.js +++ b/src/decision-engine/task-merger.js @@ -1,5 +1,10 @@ 'use strict' +/** + * @typedef {import('./types').Task} Task + * @typedef {import('./types').TaskMerger} TaskMergerAPI + */ + /** @type {TaskMergerAPI} */ const TaskMerger = { /** @@ -96,7 +101,3 @@ const TaskMerger = { } module.exports = TaskMerger -/** - * @typedef {import('./interface').Task} Task - * @typedef {import('./interface').TaskMerger} TaskMergerAPI - */ diff --git a/src/decision-engine/interface.ts b/src/decision-engine/types.d.ts similarity index 91% rename from src/decision-engine/interface.ts rename to src/decision-engine/types.d.ts index e9ab8608..1731d296 100644 --- a/src/decision-engine/interface.ts +++ b/src/decision-engine/types.d.ts @@ -3,12 +3,12 @@ export interface TaskMerger { * Given the existing tasks with the same topic, does the task add some new * information? Used to decide whether to merge the task or ignore it. */ - hasNewInfo (task:Task, tasksWithTopic:Task[]): boolean + hasNewInfo: (task: Task, tasksWithTopic: Task[]) => boolean /** * Merge the information from the task into the existing pending task. */ - merge (newTask, existingTask): void + merge: (newTask: Task, existingTask: Task) => void } export interface Task { diff --git a/src/index.js b/src/index.js index 80c03a6a..3a6af632 100644 --- a/src/index.js +++ b/src/index.js @@ -6,9 +6,20 @@ const DecisionEngine = require('./decision-engine') const Notifications = require('./notifications') const logger = require('./utils').logger const Stats = require('./stats') -const AbortController = require('native-abort-controller') +const { AbortController } = require('native-abort-controller') const { anySignal } = require('any-signal') +/** + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/bitswap').Bitswap} API + * @typedef {import('ipfs-core-types/src/bitswap').WantListEntry} WantListEntry + * @typedef {import('ipfs-core-types/src/bitswap').LedgerForPeer} LedgerForPeer + * @typedef {import('ipfs-core-types/src/block-service').Block} Block + * @typedef {import('peer-id')} PeerId + * @typedef {import('./types/message')} BitswapMessage + * @typedef {import('cids')} CID + */ + const defaultOptions = { statsEnabled: false, statsComputeThrottleTimeout: 1000, @@ -29,11 +40,13 @@ const statsKeys = [ /** * JavaScript implementation of the Bitswap 'data exchange' protocol * used by IPFS. + * + * @implements {API} */ class Bitswap { /** - * @param {LibP2P} libp2p - * @param {BlockStore} blockstore + * @param {import('libp2p')} libp2p + * @param {import('ipfs-core-types/src/block-store').BlockStore} blockstore * @param {Object} [options] * @param {boolean} [options.statsEnabled=false] * @param {number} [options.statsComputeThrottleTimeout=1000] @@ -197,9 +210,10 @@ class Bitswap { * Return the current wantlist for a given `peerId` * * @param {PeerId} peerId + * @param {AbortOptions} [_options] * @returns {Map} */ - wantlistForPeer (peerId) { + wantlistForPeer (peerId, _options) { return this.engine.wantlistForPeer(peerId) } @@ -207,7 +221,7 @@ class Bitswap { * Return ledger information for a given `peerId` * * @param {PeerId} peerId - * @returns {Object} + * @returns {null|LedgerForPeer} */ ledgerForPeer (peerId) { return this.engine.ledgerForPeer(peerId) @@ -343,9 +357,10 @@ class Bitswap { * send it to nodes that have it in their wantlist. * * @param {Block} block + * @param {AbortOptions} [_options] * @returns {Promise} */ - async put (block) { + async put (block, _options) { await this.blockstore.put(block) this._sendHaveBlockNotifications(block) } @@ -401,7 +416,7 @@ class Bitswap { /** * Get stats about the bitswap node. * - * @returns {Stats} + * @returns {import('ipfs-core-types/src/bitswap').Stats} */ stat () { return this._stats @@ -432,13 +447,3 @@ class Bitswap { } module.exports = Bitswap - -/** - * @typedef {import('libp2p')} LibP2P - * @typedef {import('./types').BlockStore} BlockStore - * @typedef {import('peer-id')} PeerId - * @typedef {import('./types/message')} BitswapMessage - * @typedef {import('ipld-block')} Block - * @typedef {import('cids')} CID - * @typedef {import('./types/wantlist/entry')} WantListEntry - */ diff --git a/src/network.js b/src/network.js index db9bbbfa..0f1cea75 100644 --- a/src/network.js +++ b/src/network.js @@ -9,15 +9,33 @@ const Message = require('./types/message') const CONSTANTS = require('./constants') const logger = require('./utils').logger +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('cids')} CID + * @typedef {import('multiaddr')} Multiaddr + * + * @typedef {Object} Connection + * @property {string} id + * @property {PeerId} remotePeer + * + * @typedef {Object} Provider + * @property {PeerId} id + * @property {Multiaddr[]} multiaddrs + * + * @typedef {Object} Stream + * @property {AsyncIterable} source + * @property {(output:AsyncIterable) => Promise} sink + */ + const BITSWAP100 = '/ipfs/bitswap/1.0.0' const BITSWAP110 = '/ipfs/bitswap/1.1.0' const BITSWAP120 = '/ipfs/bitswap/1.2.0' class Network { /** - * @param {LibP2P} libp2p - * @param {BitSwap} bitswap - * @param {Stats} stats + * @param {import('libp2p')} libp2p + * @param {import('./index')} bitswap + * @param {import('./stats')} stats * @param {Object} [options] * @param {boolean} [options.b100Only] */ @@ -71,7 +89,9 @@ class Network { this._libp2p.unhandle(this._protocols) // unregister protocol and handlers - this._libp2p.registrar.unregister(this._registrarId) + if (this._registrarId != null) { + this._libp2p.registrar.unregister(this._registrarId) + } } /** @@ -92,6 +112,9 @@ class Network { await pipe( stream, lp.decode(), + /** + * @param {AsyncIterable} source + */ async (source) => { for await (const data of source) { try { @@ -139,6 +162,8 @@ class Network { return this._libp2p.contentRouting.findProviders( cid, { + // TODO: Should this be a timeout options insetad ? + // @ts-expect-error - 'maxTimeout' does not exist in type maxTimeout: CONSTANTS.providerRequestTimeout, maxNumProviders: maxProviders, signal: options.signal @@ -172,6 +197,7 @@ class Network { * @returns {Promise} */ async provide (cid, options) { + // @ts-expect-error - contentRouting takes no options await this._libp2p.contentRouting.provide(cid, options) } @@ -224,6 +250,8 @@ class Network { throw new Error('network isn\'t running') } + // TODO: Figure out inconsistency here. + // @ts-expect-error - dial does not expects Provider return this._libp2p.dial(peer, options) } @@ -231,7 +259,7 @@ class Network { * Dial to the peer and try to use the most recent Bitswap * * @private - * @param {PeerId|Multiaddr|Provider} peer + * @param {PeerId|Multiaddr} peer */ _dialPeer (peer) { return this._libp2p.dialProtocol(peer, [BITSWAP120, BITSWAP110, BITSWAP100]) @@ -271,24 +299,3 @@ async function writeMessage (stream, msg, log) { } module.exports = Network - -/** - * @typedef {import('peer-id')} PeerId - * @typedef {import('cids')} CID - * @typedef {import('multiaddr')} Multiaddr - * @typedef {import('libp2p')} LibP2P - * @typedef {import('./stats')} Stats - * @typedef {import('./index')} BitSwap - * - * @typedef {Object} Connection - * @property {string} id - * @property {PeerId} remotePeer - * - * @typedef {Object} Provider - * @property {PeerId} id - * @property {Multiaddr[]} multiaddrs - * - * @typedef {Object} Stream - * @property {AsyncIterable} source - * @property {(output:AsyncIterable) => Promise} sink - */ diff --git a/src/notifications.js b/src/notifications.js index 7b8f2463..00332fb4 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -1,13 +1,17 @@ 'use strict' const { EventEmitter } = require('events') -const Block = require('ipld-block') +const IPLDBlock = require('ipld-block') const uint8ArrayEquals = require('uint8arrays/equals') const uint8ArrayToString = require('uint8arrays/to-string') const CONSTANTS = require('./constants') const logger = require('./utils').logger +/** + * @typedef {import('ipfs-core-types/src/block-service').Block} Block + */ + /** * @param {CID} cid */ @@ -71,6 +75,10 @@ class Notifications extends EventEmitter { this.removeListener(blockEvt, onBlock) reject(new Error(`Block for ${cid} unwanted`)) } + + /** + * @param {Block} block + */ const onBlock = (block) => { this.removeListener(unwantEvt, onUnwant) @@ -79,7 +87,7 @@ class Notifications extends EventEmitter { return reject(new Error(`Incorrect block received for ${cid}`)) } else if (cid.version !== block.cid.version || cid.codec !== block.cid.codec) { // right block but wrong version or codec - block = new Block(block.data, cid) + block = new IPLDBlock(block.data, cid) } resolve(block) diff --git a/src/stats/index.js b/src/stats/index.js index c2a057f8..cba5eb50 100644 --- a/src/stats/index.js +++ b/src/stats/index.js @@ -3,6 +3,11 @@ const { EventEmitter } = require('events') const Stat = require('./stat') +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('ipfs-core-types/src/bitswap').Stats} API + */ + /** * @typedef {[number, number, number]} AverageIntervals */ @@ -17,6 +22,9 @@ const defaultOptions = { ]) } +/** + * @implements {API} + */ class Stats extends EventEmitter { /** * @param {string[]} [initialCounters] @@ -111,6 +119,9 @@ class Stats extends EventEmitter { } } + /** + * @param {PeerId} peer + */ disconnected (peer) { const peerId = peer.toB58String() const peerStats = this._peers.get(peerId) @@ -122,7 +133,3 @@ class Stats extends EventEmitter { } module.exports = Stats - -/** - * @typedef {import('peer-id')} PeerId - */ diff --git a/src/stats/stat.js b/src/stats/stat.js index 272499b0..1ce2f1e6 100644 --- a/src/stats/stat.js +++ b/src/stats/stat.js @@ -4,6 +4,10 @@ const { EventEmitter } = require('events') const Big = require('bignumber.js').default const MovingAverage = require('moving-average') +/** + * @typedef {[string, number, number]} Op + */ + class Stats extends EventEmitter { /** * @@ -24,6 +28,7 @@ class Stats extends EventEmitter { this._stats = {} this._frequencyLastTime = Date.now() + /** @type {Record} */ this._frequencyAccumulators = {} /** @type {Record>} */ @@ -185,7 +190,3 @@ class Stats extends EventEmitter { } module.exports = Stats - -/** - * @typedef {[string, number, number]} Op - */ diff --git a/src/types.ts b/src/types.ts deleted file mode 100644 index bc5a6474..00000000 --- a/src/types.ts +++ /dev/null @@ -1,74 +0,0 @@ -import CID from 'cids' -import Block from 'ipld-block' - -export type WantBlock = 0 -export type HaveBlock = 1 -export type WantType = WantBlock | HaveBlock - -export type BlockData = { - prefix: Uint8Array - data: Uint8Array -} - -export type Have = 0 -export type DontHave = 1 -export type BlockPresenceType = Have | DontHave - -export type BlockPresence = { - cid: Uint8Array - type: BlockPresenceType -} - -export type Entry = { - block: Uint8Array - priority: number - cancel: boolean - wantType?: WantType - sendDontHave?: boolean -} - -export type Message110 = { - wantlist: WantList - blockPresences: BlockPresence[] - payload: BlockData[] - - pendingBytes?: number -} - -export type Message100 = { - wantlist: WantList - blocks: Uint8Array[] - - pendingBytes?: number -} - -export type WantList = { - entries: Entry[] - full?: boolean -} - -export type AbortOptions = { - signal: AbortSignal -} - -export interface BlockStore { - has(cid:CID, options?:AbortOptions):Promise - get(cid:CID, options?:AbortOptions):Promise - put(block:Block, options?:AbortOptions):Promise - putMany(blocks:AsyncIterable|Iterable, options?:AbortOptions):AsyncIterable -} - -export type MessageProto = { - encode(value:any): Uint8Array - decode(bytes: Uint8Array): any - BlockPresenceType: { - Have: Have, - DontHave: DontHave - }, - Wantlist: { - WantType: { - Block: WantBlock - Have: HaveBlock - } - } -} diff --git a/src/types/message/entry.js b/src/types/message/entry.js index ca02d473..65a509b6 100644 --- a/src/types/message/entry.js +++ b/src/types/message/entry.js @@ -4,9 +4,9 @@ const WantlistEntry = require('../wantlist').Entry module.exports = class BitswapMessageEntry { /** - * @param {CID} cid + * @param {import('cids')} cid * @param {number} priority - * @param {WantType} wantType + * @param {import('./message.proto').WantType} wantType * @param {boolean} [cancel] * @param {boolean} [sendDontHave] */ @@ -45,6 +45,9 @@ module.exports = class BitswapMessageEntry { return `BitswapMessageEntry ${cidStr} ` } + /** + * @param {this} other + */ equals (other) { return (this.cancel === other.cancel) && (this.sendDontHave === other.sendDontHave) && @@ -52,8 +55,3 @@ module.exports = class BitswapMessageEntry { this.entry.equals(other.entry) } } - -/** - * @typedef {import('../../types').WantType} WantType - * @typedef {import('cids')} CID - */ diff --git a/src/types/message/index.js b/src/types/message/index.js index 9f780ab1..9d2e1ead 100644 --- a/src/types/message/index.js +++ b/src/types/message/index.js @@ -1,8 +1,9 @@ 'use strict' -const Block = require('ipld-block') +const IPLDBlock = require('ipld-block') const CID = require('cids') const { getName } = require('multicodec') +// @ts-ignore const vd = require('varint-decoder') const multihashing = require('multihashing-async') const { isMapEqual } = require('../../utils') @@ -18,11 +19,10 @@ class BitswapMessage { /** @type {Map} */ this.wantlist = new Map() - /** @type {Map} - */ + /** @type {Map} */ this.blocks = new Map() - /** @type {Map} */ + /** @type {Map} */ this.blockPresences = new Map() this.pendingBytes = 0 } @@ -37,7 +37,7 @@ class BitswapMessage { * * @param {CID} cid * @param {number} priority - * @param {WantType} [wantType] + * @param {import('./message.proto').WantType} [wantType] * @param {boolean} [cancel] * @param {boolean} [sendDontHave] * @returns {void} @@ -72,7 +72,7 @@ class BitswapMessage { } /** - * @param {Block} block + * @param {import('ipfs-core-types/src/block-service').Block} block * @returns {void} */ addBlock (block) { @@ -123,7 +123,7 @@ class BitswapMessage { * @returns {Uint8Array} */ serializeToBitswap100 () { - /** @type {Message100} */ + /** @type {import('./message.proto').Message100} */ const msg = { wantlist: { entries: Array.from(this.wantlist.values()).map((entry) => { @@ -152,7 +152,7 @@ class BitswapMessage { * @returns {Uint8Array} */ serializeToBitswap110 () { - /** @type {Message110} */ + /** @type {import('./message.proto').Message110} */ const msg = { wantlist: { entries: Array.from(this.wantlist.values()).map((entry) => { @@ -166,7 +166,8 @@ class BitswapMessage { }) }, blockPresences: [], - payload: [] + payload: [], + pendingBytes: this.pendingBytes } if (this.full) { @@ -203,9 +204,8 @@ class BitswapMessage { this.pendingBytes !== other.pendingBytes || !isMapEqual(this.wantlist, other.wantlist) || !isMapEqual(this.blocks, other.blocks) || - // @TODO - Is this a bug ? `isMapEqual` only compare maps of blocks and - // wants - // @ts-ignore + // @TODO - Is this a bug ? + // @ts-expect-error - isMap equals map values to be objects not numbers !isMapEqual(this.blockPresences, other.blockPresences) ) { return false @@ -257,7 +257,7 @@ BitswapMessage.deserialize = async (raw) => { await Promise.all(decoded.blocks.map(async (b) => { const hash = await multihashing(b, 'sha2-256') const cid = new CID(hash) - msg.addBlock(new Block(b, cid)) + msg.addBlock(new IPLDBlock(b, cid)) })) return msg } @@ -275,7 +275,7 @@ BitswapMessage.deserialize = async (raw) => { // const hashLen = values[3] // We haven't need to use this so far const hash = await multihashing(p.data, hashAlg) const cid = new CID(cidVersion, getName(multicodec), hash) - msg.addBlock(new Block(p.data, cid)) + msg.addBlock(new IPLDBlock(p.data, cid)) })) msg.setPendingBytes(decoded.pendingBytes) return msg @@ -297,22 +297,15 @@ BitswapMessage.blockPresenceSize = (cid) => { BitswapMessage.Entry = Entry BitswapMessage.WantType = { - /** @type {import('../../types').WantBlock} */ + /** @type {import('./message.proto').WantBlock} */ Block: Message.Wantlist.WantType.Block, - /** @type {import('../../types').HaveBlock} */ + /** @type {import('./message.proto').HaveBlock} */ Have: Message.Wantlist.WantType.Have } BitswapMessage.BlockPresenceType = { - /** @type {import('../../types').Have} */ + /** @type {import('./message.proto').Have} */ Have: Message.BlockPresenceType.Have, - /** @type {import('../../types').DontHave} */ + /** @type {import('./message.proto').DontHave} */ DontHave: Message.BlockPresenceType.DontHave } module.exports = BitswapMessage - -/** - * @typedef {import('../../types').WantType} WantType - * @typedef {import('../../types').Message110} Message110 - * @typedef {import('../../types').Message100} Message100 - * @typedef {import('../../types').BlockPresenceType} BlockPresenceType - */ diff --git a/src/types/message/message.proto.d.ts b/src/types/message/message.proto.d.ts new file mode 100644 index 00000000..312df051 --- /dev/null +++ b/src/types/message/message.proto.d.ts @@ -0,0 +1,73 @@ + +export interface MessageProto { + decode: (bytes: Uint8Array) => MessageData + encode: (value: Message100|Message110) => Uint8Array + BlockPresenceType: { + Have: Have + DontHave: DontHave + } + Wantlist: { + WantType: { + Block: WantBlock + Have: HaveBlock + } + } +} + +export interface MessageData { + wantlist?: WantList + blockPresences: BlockPresence[] + + blocks: Uint8Array[] + payload: Block[] + + pendingBytes: number +} + +export interface Message110 { + wantlist: WantList + blockPresences: BlockPresence[] + + payload: Block[] + pendingBytes: number +} + +export interface Message100 { + wantlist: WantList + + blocks: Uint8Array[] +} + +export interface BlockPresence { + cid: Uint8Array + type: BlockPresenceType +} + +export interface WantList { + entries: Entry[] + full?: boolean +} + +export type WantBlock = 0 +export type HaveBlock = 1 +export type WantType = WantBlock | HaveBlock + +export type Have = 0 +export type DontHave = 1 +export type BlockPresenceType = Have | DontHave + +export interface Entry { + block: Uint8Array + priority: number + cancel: boolean + wantType?: WantType + sendDontHave?: boolean +} + +export interface Block { + prefix: Uint8Array + data: Uint8Array +} + +declare var Message: MessageProto +export { Message } diff --git a/src/types/message/message.proto.js b/src/types/message/message.proto.js index 23748d25..03e0d114 100644 --- a/src/types/message/message.proto.js +++ b/src/types/message/message.proto.js @@ -1,8 +1,9 @@ +// @ts-nocheck 'use strict' + const protons = require('protons') // from: https://github.com/ipfs/go-ipfs/blob/master/exchange/bitswap/message/pb/message.proto -/** @type {{Message: import('../../types').MessageProto}} */ module.exports = protons(` message Message { message Wantlist { diff --git a/src/types/wantlist/entry.js b/src/types/wantlist/entry.js index 09ac5539..2e46f188 100644 --- a/src/types/wantlist/entry.js +++ b/src/types/wantlist/entry.js @@ -1,10 +1,17 @@ 'use strict' +/** + * @typedef {import('ipfs-core-types/src/bitswap').WantListEntry} API + */ + +/** + * @implements {API} + */ class WantListEntry { /** - * @param {CID} cid + * @param {import('cids')} cid * @param {number} priority - * @param {WantType} [wantType] + * @param {import('../message/message.proto').WantType} wantType */ constructor (cid, priority, wantType) { // Keep track of how many requests we have for this key @@ -34,20 +41,17 @@ class WantListEntry { } /** - * @param {WantListEntry} other + * @param {API} other * @returns {boolean} */ equals (other) { + // @ts-expect-error _refCounter is not specified by the interface return (this._refCounter === other._refCounter) && this.cid.equals(other.cid) && this.priority === other.priority && + // @ts-expect-error - wantType is not specified by the interface this.wantType === other.wantType } } module.exports = WantListEntry - -/** - * @typedef {import('../../types').WantType} WantType - * @typedef {import('cids')} CID - */ diff --git a/src/types/wantlist/index.js b/src/types/wantlist/index.js index e9cc9f21..54408112 100644 --- a/src/types/wantlist/index.js +++ b/src/types/wantlist/index.js @@ -3,10 +3,14 @@ const { sortBy } = require('../../utils') const Entry = require('./entry') +/** + * @typedef {import('cids')} CID + */ + class Wantlist { /** * - * @param {Stats} [stats] + * @param {import('../../stats')} [stats] */ constructor (stats) { /** @type {Map} */ @@ -21,7 +25,7 @@ class Wantlist { /** * @param {CID} cid * @param {number} priority - * @param {WantType} [wantType] + * @param {import('../message/message.proto').WantType} wantType * @returns {void} */ add (cid, priority, wantType) { @@ -93,8 +97,8 @@ class Wantlist { } sortedEntries () { - // @ts-ignore - Property 'key' does not exist on type 'WantListEntry'.ts(2339) // TODO: Figure out if this is an actual bug. + // @ts-expect-error - Property 'key' does not exist on type 'WantListEntry' return new Map(sortBy(o => o[1].key, Array.from(this.set.entries()))) } @@ -111,9 +115,3 @@ class Wantlist { Wantlist.Entry = Entry module.exports = Wantlist - -/** - * @typedef {import('cids')} CID - * @typedef {import('../../types').WantType} WantType - * @typedef {import('../../stats')} Stats - */ diff --git a/src/utils/index.js b/src/utils/index.js index 9aca6792..9f66f2e0 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -6,7 +6,7 @@ const uint8ArrayEquals = require('uint8arrays/equals') /** * Creates a logger for the given subsystem * - * @param {PeerId} [id] + * @param {import('peer-id')} [id] * @param {string} [subsystem] */ const logger = (id, subsystem) => { @@ -136,7 +136,7 @@ const isMapEqual = (a, b) => { return false } // Support Blocks - if (valueA.data && !uint8ArrayEquals(valueA.data, valueB.data)) { + if (valueA.data && !(valueB.data && uint8ArrayEquals(valueA.data, valueB.data))) { return false } } @@ -153,8 +153,3 @@ module.exports = { sortBy, isMapEqual } - -/** - * @typedef {import('peer-id')} PeerId - * @typedef {import('../types/message')} BitswapMessageEntry - */ diff --git a/src/utils/sorted-map.js b/src/utils/sorted-map.js index d79315c2..bfada300 100644 --- a/src/utils/sorted-map.js +++ b/src/utils/sorted-map.js @@ -172,13 +172,19 @@ class SortedMap extends Map { yield * this.entries() } + /** + * @template This + * @param {(entry:[Key, Value]) => void} cb + * @param {This} [thisArg] + */ + // @ts-expect-error - Callback in Map forEach is (V, K, Map) => void forEach (cb, thisArg) { if (!cb) { return } for (const k of this._keys) { - cb.apply(thisArg, [[k, this.get(k)]]) + cb.apply(thisArg, [[k, /** @type {Value} */(this.get(k))]]) } } diff --git a/src/want-manager/index.js b/src/want-manager/index.js index c27856a9..30a26c5d 100644 --- a/src/want-manager/index.js +++ b/src/want-manager/index.js @@ -6,14 +6,20 @@ const CONSTANTS = require('../constants') const MsgQueue = require('./msg-queue') const logger = require('../utils').logger +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('cids')} CID + */ + module.exports = class WantManager { /** * * @param {PeerId} peerId - * @param {Network} network - * @param {Stats} stats + * @param {import('../network')} network + * @param {import('../stats')} stats */ constructor (peerId, network, stats) { + /** @type {Map} */ this.peers = new Map() this.wantlist = new Wantlist(stats) @@ -24,6 +30,12 @@ module.exports = class WantManager { this._log = logger(peerId, 'want') } + /** + * @private + * @param {CID[]} cids + * @param {boolean} cancel + * @param {boolean} [force] + */ _addEntries (cids, cancel, force) { const entries = cids.map((cid, i) => { return new Message.Entry(cid, CONSTANTS.kMaxPriority - i, Message.WantType.Block, cancel) @@ -33,12 +45,14 @@ module.exports = class WantManager { // add changes to our wantlist if (e.cancel) { if (force) { - this.wantlist.removeForce(e.cid) + this.wantlist.removeForce(e.cid.toString()) } else { this.wantlist.remove(e.cid) } } else { this._log('adding to wl') + // TODO: Figure out the wantType + // @ts-expect-error - requires wantType this.wantlist.add(e.cid, e.priority) } }) @@ -49,6 +63,10 @@ module.exports = class WantManager { } } + /** + * @private + * @param {PeerId} peerId + */ _startPeerHandler (peerId) { let mq = this.peers.get(peerId.toB58String()) @@ -72,6 +90,10 @@ module.exports = class WantManager { return mq } + /** + * @private + * @param {PeerId} peerId + */ _stopPeerHandler (peerId) { const mq = this.peers.get(peerId.toB58String()) @@ -87,7 +109,13 @@ module.exports = class WantManager { this.peers.delete(peerId.toB58String()) } - // add all the cids to the wantlist + /** + * add all the cids to the wantlist + * + * @param {CID[]} cids + * @param {Object} [options] + * @param {AbortSignal} [options.signal] + */ wantBlocks (cids, options = {}) { this._addEntries(cids, false) @@ -98,27 +126,46 @@ module.exports = class WantManager { } } - // remove blocks of all the given keys without respecting refcounts + /** + * Remove blocks of all the given keys without respecting refcounts + * + * @param {CID[]} cids + */ unwantBlocks (cids) { this._log('unwant blocks: %s', cids.length) this._addEntries(cids, true, true) } - // cancel wanting all of the given keys + /** + * Cancel wanting all of the given keys + * + * @param {CID[]} cids + */ cancelWants (cids) { this._log('cancel wants: %s', cids.length) this._addEntries(cids, true) } - // Returns a list of all currently connected peers + /** + * Returns a list of all currently connected peers + * + * @returns {string[]} + */ connectedPeers () { return Array.from(this.peers.keys()) } + /** + * @param {PeerId} peerId + */ connected (peerId) { this._startPeerHandler(peerId) } + /** + * @param {PeerId} peerId + */ + disconnected (peerId) { this._stopPeerHandler(peerId) } @@ -130,9 +177,3 @@ module.exports = class WantManager { this.peers.forEach((mq) => this.disconnected(mq.peerId)) } } - -/** - * @typedef {import('peer-id')} PeerId - * @typedef {import('../network')} Network - * @typedef {import('../stats')} Stats - */ diff --git a/src/want-manager/msg-queue.js b/src/want-manager/msg-queue.js index 0b576740..c3f51bd1 100644 --- a/src/want-manager/msg-queue.js +++ b/src/want-manager/msg-queue.js @@ -1,14 +1,20 @@ 'use strict' +// @ts-ignore const debounce = require('just-debounce-it') const Message = require('../types/message') const logger = require('../utils').logger const { wantlistSendDebounceMs } = require('../constants') +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('cids')} CID + * @typedef {import('../network')} Network + */ + module.exports = class MsgQueue { /** - * * @param {PeerId} selfPeerId * @param {PeerId} otherPeerId * @param {Network} network @@ -18,11 +24,19 @@ module.exports = class MsgQueue { this.network = network this.refcnt = 1 + /** + * @private + * @type {{cid:CID, priority:number, cancel?:boolean}[]} + */ this._entries = [] + /** @private */ this._log = logger(selfPeerId, 'msgqueue') this.sendEntries = debounce(this._sendEntries.bind(this), wantlistSendDebounceMs) } + /** + * @param {Message} msg + */ addMessage (msg) { if (msg.empty) { return @@ -31,11 +45,17 @@ module.exports = class MsgQueue { this.send(msg) } + /** + * @param {{cid:CID, priority:number}[]} entries + */ addEntries (entries) { this._entries = this._entries.concat(entries) this.sendEntries() } + /** + * @private + */ _sendEntries () { if (!this._entries.length) { return @@ -53,6 +73,9 @@ module.exports = class MsgQueue { this.addMessage(msg) } + /** + * @param {Message} msg + */ async send (msg) { try { await this.network.connectTo(this.peerId) @@ -69,8 +92,3 @@ module.exports = class MsgQueue { }) } } - -/** - * @typedef {import('peer-id')} PeerId - * @typedef {import('../network')} Network - */ diff --git a/test/bitswap-mock-internals.js b/test/bitswap-mock-internals.js index 185e7780..d014b9e1 100644 --- a/test/bitswap-mock-internals.js +++ b/test/bitswap-mock-internals.js @@ -11,7 +11,7 @@ const Message = require('../src/types/message') const Bitswap = require('../src') const CID = require('cids') const Block = require('ipld-block') -const AbortController = require('native-abort-controller') +const { AbortController } = require('native-abort-controller') const delay = require('delay') const createTempRepo = require('./utils/create-temp-repo-nodejs') diff --git a/test/notifications.spec.js b/test/notifications.spec.js index d2a8670e..edb6f0d1 100644 --- a/test/notifications.spec.js +++ b/test/notifications.spec.js @@ -4,7 +4,7 @@ const { expect } = require('aegir/utils/chai') const CID = require('cids') const Block = require('ipld-block') -const AbortController = require('native-abort-controller') +const { AbortController } = require('native-abort-controller') const uint8ArrayToString = require('uint8arrays/to-string') const Notifications = require('../src/notifications') diff --git a/test/types/message.spec.js b/test/types/message.spec.js index 59eda5c8..7dcc84af 100644 --- a/test/types/message.spec.js +++ b/test/types/message.spec.js @@ -5,7 +5,7 @@ const { expect } = require('aegir/utils/chai') const CID = require('cids') const uint8ArrayFromString = require('uint8arrays/from-string') const uint8ArrayEquals = require('uint8arrays/equals') -const loadFixture = require('aegir/fixtures') +const loadFixture = require('aegir/utils/fixtures') const testDataPath = 'test/fixtures/serialized-from-go' const rawMessageFullWantlist = loadFixture(testDataPath + '/bitswap110-message-full-wantlist') const rawMessageOneBlock = loadFixture(testDataPath + '/bitswap110-message-one-block') diff --git a/test/utils/connect-all.js b/test/utils/connect-all.js index 40a34e0d..75471ec5 100644 --- a/test/utils/connect-all.js +++ b/test/utils/connect-all.js @@ -1,7 +1,12 @@ 'use strict' +// @ts-ignore const without = require('lodash.without') +/** + * + * @param {any[]} nodes + */ module.exports = async (nodes) => { for (const node of nodes) { for (const otherNode of without(nodes, node)) { diff --git a/test/utils/create-libp2p-node.js b/test/utils/create-libp2p-node.js index 98649980..32eefa0a 100644 --- a/test/utils/create-libp2p-node.js +++ b/test/utils/create-libp2p-node.js @@ -1,15 +1,21 @@ 'use strict' +// @ts-ignore const TCP = require('libp2p-tcp') +// @ts-ignore const MPLEX = require('libp2p-mplex') +// @ts-ignore const SECIO = require('libp2p-secio') const libp2p = require('libp2p') const KadDHT = require('libp2p-kad-dht') const PeerId = require('peer-id') - +// @ts-ignore const defaultsDeep = require('@nodeutils/defaults-deep') class Node extends libp2p { + /** + * @param {Partial & import('libp2p').constructorOptions & { DHT?: boolean}} _options + */ constructor (_options) { const defaults = { modules: { diff --git a/test/utils/create-temp-repo-browser.js b/test/utils/create-temp-repo-browser.js index c7b23c76..973bec2f 100644 --- a/test/utils/create-temp-repo-browser.js +++ b/test/utils/create-temp-repo-browser.js @@ -1,6 +1,7 @@ /* global self */ 'use strict' +// @ts-ignore const IPFSRepo = require('ipfs-repo') // @ts-ignore @@ -10,6 +11,7 @@ async function createTempRepo () { const date = Date.now().toString() const path = `/bitswap-tests-${date}-${Math.random()}` + /** @type {import('ipfs-core-types/src/repo').Repo & { teardown: () => void}} */ const repo = new IPFSRepo(path) await repo.init({}) await repo.open() diff --git a/test/utils/create-temp-repo-nodejs.js b/test/utils/create-temp-repo-nodejs.js index 519ab085..c7b48240 100644 --- a/test/utils/create-temp-repo-nodejs.js +++ b/test/utils/create-temp-repo-nodejs.js @@ -1,20 +1,26 @@ 'use strict' +// @ts-ignore const IPFSRepo = require('ipfs-repo') const pathJoin = require('path').join const os = require('os') +// @ts-ignore const ncp = require('ncp') +// @ts-ignore const rimraf = require('rimraf') +// @ts-ignore const promisify = require('promisify-es6') const baseRepo = pathJoin(__dirname, '../fixtures/repo') +const asyncNcp = promisify(ncp) async function createTempRepo () { const date = Date.now().toString() const path = pathJoin(os.tmpdir(), `bitswap-tests-${date}-${Math.random()}`) - await promisify(ncp)(baseRepo, path) + await asyncNcp(baseRepo, path) + /** @type {import('ipfs-core-types/src/repo').Repo & { teardown: () => Promise}} */ const repo = new IPFSRepo(path) repo.teardown = async () => { diff --git a/test/utils/distribution-test.js b/test/utils/distribution-test.js index d1e68d71..29f890f1 100644 --- a/test/utils/distribution-test.js +++ b/test/utils/distribution-test.js @@ -1,5 +1,7 @@ 'use strict' +/** @type {(n:number) => any[]} */ +// @ts-ignore const range = require('lodash.range') const { expect } = require('aegir/utils/chai') @@ -7,6 +9,13 @@ const createBitswap = require('./create-bitswap') const makeBlock = require('./make-block') const connectAll = require('./connect-all') +/** + * + * @param {number} instanceCount + * @param {number} blockCount + * @param {number} repeats + * @param {*} events + */ module.exports = async (instanceCount, blockCount, repeats, events) => { let pendingRepeats = repeats diff --git a/test/utils/helpers.js b/test/utils/helpers.js index a9aad9a1..d6545c2a 100644 --- a/test/utils/helpers.js +++ b/test/utils/helpers.js @@ -1,12 +1,20 @@ 'use strict' +// @ts-ignore const range = require('lodash.range') const { expect } = require('aegir/utils/chai') +/** + * @param {number} n + */ exports.orderedFinish = (n) => { const r = range(1, n + 1) + /** @type {number[]} */ const finishes = [] + /** + * @param {number} i + */ const output = (i) => { finishes.push(i) } diff --git a/test/utils/make-block.js b/test/utils/make-block.js index 81dfc3fc..5820dd9b 100644 --- a/test/utils/make-block.js +++ b/test/utils/make-block.js @@ -3,11 +3,19 @@ const multihashing = require('multihashing-async') const CID = require('cids') const Block = require('ipld-block') +// @ts-ignore const randomBytes = require('iso-random-stream/src/random') +// @ts-ignore const range = require('lodash.range') const uint8ArrayFromString = require('uint8arrays/from-string') +// @ts-ignore const { v4: uuid } = require('uuid') +/** + * @param {number} count + * @param {number} [size] + * @returns {Promise} + */ module.exports = async (count, size) => { const blocks = await Promise.all( range(count || 1).map(async () => { diff --git a/test/utils/make-peer-id.js b/test/utils/make-peer-id.js index 3d74c33c..70811e73 100644 --- a/test/utils/make-peer-id.js +++ b/test/utils/make-peer-id.js @@ -6,6 +6,10 @@ async function makePeerId () { return (await makePeerIds(1))[0] } +/** + * @param {number} count + * @returns {Promise} + */ async function makePeerIds (count) { const peerIds = await Promise.all([...new Array(count || 1)].map(() => { return PeerId.create({ bits: 512 }) diff --git a/test/wantmanager/msg-queue.spec.js b/test/wantmanager/msg-queue.spec.js index 1860189d..7898ee33 100644 --- a/test/wantmanager/msg-queue.spec.js +++ b/test/wantmanager/msg-queue.spec.js @@ -13,7 +13,9 @@ const { } = require('../utils/mocks') describe('MessageQueue', () => { + /** @type {PeerId[]} */ let peerIds + /** @type {CID[]} */ let cids before(async () => { diff --git a/tsconfig.json b/tsconfig.json index 77830df2..0cbdc31a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,6 @@ "outDir": "dist" }, "include": [ - "src", - "test" + "src" ] } From 9a0708cb4bce5f906082cb27de74e16a708d74f0 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 9 Mar 2021 10:56:58 +0000 Subject: [PATCH 10/13] chore: remove gh urls --- package.json | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index b0d668fd..c976b2f9 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,9 @@ }, "eslintConfig": { "extends": "ipfs", - "ignorePatterns": ["scripts/*"] + "ignorePatterns": [ + "scripts/*" + ] }, "files": [ "dist", @@ -65,7 +67,7 @@ "assert": "^2.0.0", "benchmark": "^2.1.4", "delay": "^5.0.0", - "ipfs-repo": "^7.0.0", + "ipfs-repo": "^9.0.0", "ipfs-utils": "^6.0.1", "iso-random-stream": "^1.1.1", "it-all": "^1.0.5", @@ -95,13 +97,13 @@ "bignumber.js": "^9.0.0", "cids": "^1.1.6", "debug": "^4.2.0", - "ipfs-core-types": "^0.3.0", + "ipfs-core-types": "^0.3.1", "ipld-block": "^0.11.0", "it-length-prefixed": "^3.1.0", "it-pipe": "^1.1.0", "just-debounce-it": "^1.1.0", "libp2p-interfaces": "^0.8.3", - "moving-average": "git://github.com/gozala/moving-average#types", + "moving-average": "^1.0.1", "multicodec": "^3.0.1", "multihashing-async": "^2.1.2", "native-abort-controller": "^1.0.3", From 76036fa9d76f4fdb8aac35865c09ce71ad554dc4 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 9 Mar 2021 11:08:52 +0000 Subject: [PATCH 11/13] chore: update node version --- .github/workflows/main.yml | 2 +- .github/workflows/typecheck.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1cfde4ec..7b3c2f61 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,7 +44,7 @@ jobs: - uses: actions/checkout@v2 - uses: microsoft/playwright-github-action@v1 - run: npm install - - run: npx aegir test -t browser -t webworker --bail # add --cov later when its fixed + - run: npx aegir test -t browser -t webworker --bail # add --cov later when its fixed - uses: codecov/codecov-action@v1 test-firefox: needs: check diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index 61463746..0da155e6 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [12.x] + node-version: [15.x] steps: - uses: actions/checkout@v1 - name: Use Node.js ${{ matrix.node-version }} From 71c3d15f55de58435a4efc6145f7f7740abae58b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 9 Mar 2021 11:21:01 +0000 Subject: [PATCH 12/13] chore: update bundle size config param --- .aegir.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.aegir.js b/.aegir.js index 74902375..f5a5d320 100644 --- a/.aegir.js +++ b/.aegir.js @@ -27,7 +27,7 @@ module.exports = { } }, build: { - bundlesize: { maxSize: '68kB' }, + bundlesizeMax: '61kB', config: esbuild } } From bdcc596c818b573727b9410d6a61396680a3f675 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 9 Mar 2021 11:52:00 +0000 Subject: [PATCH 13/13] chore: add temporary exit --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7b3c2f61..e885421e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,7 +44,7 @@ jobs: - uses: actions/checkout@v2 - uses: microsoft/playwright-github-action@v1 - run: npm install - - run: npx aegir test -t browser -t webworker --bail # add --cov later when its fixed + - run: npx aegir test -t browser -t webworker --bail -- --exit # TODO remove - https://mochajs.org/#-exit - uses: codecov/codecov-action@v1 test-firefox: needs: check