From 579311b09228cd175a07e6f980003445f860be4f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 25 Apr 2019 16:41:09 +0100 Subject: [PATCH 01/11] feat: convert to async/await --- README.md | 276 ++-- package.json | 16 +- src/dir-flat.js | 53 - src/dir-hamt-sharded.js | 203 --- src/index.js | 145 +- src/object.js | 34 - src/raw.js | 62 - src/resolve.js | 101 -- src/resolvers/dag-cbor.js | 53 + src/resolvers/index.js | 21 + src/resolvers/raw.js | 66 + src/resolvers/unixfs-v1/content/directory.js | 17 + src/{ => resolvers/unixfs-v1/content}/file.js | 107 +- .../content/hamt-sharded-directory.js | 26 + src/resolvers/unixfs-v1/content/raw.js | 43 + src/resolvers/unixfs-v1/index.js | 81 + src/{ => utils}/extract-data-from-block.js | 0 src/utils/find-cid-in-shard.js | 101 ++ test/exporter-sharded.spec.js | 460 ++---- test/exporter-subtree.spec.js | 282 ++-- test/exporter.spec.js | 1339 ++++++----------- test/helpers/create-shard.js | 33 + test/helpers/dag-pb.js | 30 + test/helpers/random-bytes.js | 22 - 24 files changed, 1413 insertions(+), 2158 deletions(-) delete mode 100644 src/dir-flat.js delete mode 100644 src/dir-hamt-sharded.js delete mode 100644 src/object.js delete mode 100644 src/raw.js delete mode 100644 src/resolve.js create mode 100644 src/resolvers/dag-cbor.js create mode 100644 src/resolvers/index.js create mode 100644 src/resolvers/raw.js create mode 100644 src/resolvers/unixfs-v1/content/directory.js rename src/{ => resolvers/unixfs-v1/content}/file.js (67%) create mode 100644 src/resolvers/unixfs-v1/content/hamt-sharded-directory.js create mode 100644 src/resolvers/unixfs-v1/content/raw.js create mode 100644 src/resolvers/unixfs-v1/index.js rename src/{ => utils}/extract-data-from-block.js (100%) create mode 100644 src/utils/find-cid-in-shard.js create mode 100644 test/helpers/create-shard.js create mode 100644 test/helpers/dag-pb.js delete mode 100644 test/helpers/random-bytes.js diff --git a/README.md b/README.md index b893980..da2992d 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,21 @@ ## Table of Contents -- [Install](#install) -- [Usage](#usage) - - [Example](#example) - - [API](#api) - - [exporter(cid, ipld)](#exportercid-ipld-options) -- [Contribute](#contribute) -- [License](#license) +- [ipfs-unixfs-exporter](#ipfs-unixfs-exporter) + - [Lead Maintainer](#lead-maintainer) + - [Table of Contents](#table-of-contents) + - [Install](#install) + - [Usage](#usage) + - [Example](#example) + - [API](#api) + - [exporter(cid, ipld)](#exportercid-ipld) + - [UnixFS V1 entries](#unixfs-v1-entries) + - [Raw entries](#raw-entries) + - [CBOR entries](#cbor-entries) + - [`entry.content({ offset, length })`](#entrycontent-offset-length) + - [exporter.path(cid, ipld)](#exporterpathcid-ipld) + - [Contribute](#contribute) + - [License](#license) ## Install @@ -38,29 +46,41 @@ ### Example ```js -// Create an export source pull-stream cid or ipfs path you want to export and a -// to fetch the file from +// import a file and export it again +const importer = require('ipfs-unixfs-importer') const exporter = require('ipfs-unixfs-exporter') -const pull = require('pull-stream/pull') -const { stdout } = require('pull-stdio') - -const options = {} - -pull( - exporter(cid, ipld, options), - collect((error, files) => { - if (error) { - // ...handle error - } - - // Set up a pull stream that sends the file content to process.stdout - pull( - // files[0].content is a pull-stream that contains the bytes of the file - files[0].content, - stdout() - ) - }) -) + +const files = [] + +for await (const file of importer([{ + path: '/foo/bar.txt', + content: Buffer.from(0, 1, 2, 3) +}], ipld)) { + files.push(file) +} + +console.info(files[0].cid) // Qmbaz + +const entry = await exporter(files[0].cid, ipld) + +console.info(entry.cid) // Qmqux +console.info(entry.path) // Qmbaz/foo/bar.txt +console.info(entry.name) // bar.txt +console.info(entry.unixfs.fileSize()) // 4 + +// stream content from unixfs node +const bytes = [] + +for await (const buf of entry.content({ + offset: 0, // optional offset + length: 4 // optional length +})) { + bytes.push(buf) +} + +const content = Buffer.concat(bytes) + +console.info(content) // 0, 1, 2, 3 ``` #### API @@ -69,124 +89,126 @@ pull( const exporter = require('ipfs-unixfs-exporter') ``` -### exporter(cid, ipld, options) +### exporter(cid, ipld) -Uses the given [dag API][] or an [ipld-resolver instance][] to fetch an IPFS [UnixFS][] object(s) by their CID. +Uses the given [js-ipld instance][] to fetch an IPFS node by it's CID. -Creates a new pull stream that outputs objects of the form +Returns a Promise which resolves to an `entry`. -```js +#### UnixFS V1 entries + +Entries with a `dag-pb` codec `CID` return UnixFS V1 entries: + +```javascript { - path: 'a name', - content: + name: 'foo.txt', + path: 'Qmbar/foo.txt', + cid: CID, // see https://github.com/multiformats/js-cid + node: DAGNode, // see https://github.com/ipld/js-ipld-dag-pb + content: function, // returns an async iterator + unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs } ``` -#### `offset` and `length` +If the entry is a file, `entry.content()` returns an async iterator that emits buffers containing the file content: -`offset` and `length` arguments can optionally be passed to the exporter function. These will cause the returned stream to only emit bytes starting at `offset` and with length of `length`. +```javascript +for await (const chunk of entry.content()) { + // chunk is a Buffer +} +``` -See [the tests](test/exporter.js) for examples of using these arguments. +If the entry is a directory or hamt shard, `entry.content()` returns further `entry` objects: -```js -const exporter = require('ipfs-unixfs-exporter') -const pull = require('pull-stream') -const drain = require('pull-stream/sinks/drain') - -pull( - exporter(cid, ipld, { - offset: 0, - length: 10 - }) - drain((file) => { - // file.content is a pull stream containing only the first 10 bytes of the file - }) -) +```javascript +for await (const entry of dir.content()) { + console.info(entry.name) +} ``` -### `fullPath` +#### Raw entries -If specified the exporter will emit an entry for every path component encountered. +Entries with a `raw` codec `CID` return raw entries: ```javascript -const exporter = require('ipfs-unixfs-exporter') -const pull = require('pull-stream') -const collect = require('pull-stream/sinks/collect') - -pull( - exporter('QmFoo.../bar/baz.txt', ipld, { - fullPath: true - }) - collect((err, files) => { - console.info(files) - - // [{ - // depth: 0, - // name: 'QmFoo...', - // path: 'QmFoo...', - // size: ... - // cid: CID - // content: undefined - // type: 'dir' - // }, { - // depth: 1, - // name: 'bar', - // path: 'QmFoo.../bar', - // size: ... - // cid: CID - // content: undefined - // type: 'dir' - // }, { - // depth: 2, - // name: 'baz.txt', - // path: 'QmFoo.../bar/baz.txt', - // size: ... - // cid: CID - // content: - // type: 'file' - // }] - // - }) -) +{ + name: 'foo.txt', + path: 'Qmbar/foo.txt', + cid: CID, // see https://github.com/multiformats/js-cid + node: Buffer, // see https://nodejs.org/api/buffer.html + content: function, // returns an async iterator +} +``` + +`entry.content()` returns an async iterator that emits buffers containing the node content: + +```javascript +for await (const chunk of entry.content()) { + // chunk is a Buffer +} ``` -### `maxDepth` +#### CBOR entries -If specified the exporter will only emit entries up to the specified depth. +Entries with a `dag-cbor` codec `CID` return JavaScript object entries: ```javascript -const exporter = require('ipfs-unixfs-exporter') -const pull = require('pull-stream') -const collect = require('pull-stream/sinks/collect') - -pull( - exporter('QmFoo.../bar/baz.txt', ipld, { - fullPath: true, - maxDepth: 1 - }) - collect((err, files) => { - console.info(files) - - // [{ - // depth: 0, - // name: 'QmFoo...', - // path: 'QmFoo...', - // size: ... - // cid: CID - // content: undefined - // type: 'dir' - // }, { - // depth: 1, - // name: 'bar', - // path: 'QmFoo.../bar', - // size: ... - // cid: CID - // content: undefined - // type: 'dir' - // }] - // - }) -) +{ + name: 'foo.txt', + path: 'Qmbar/foo.txt', + cid: CID, // see https://github.com/multiformats/js-cid + node: Object, // see https://github.com/ipld/js-ipld-dag-cbor +} +``` + +There is no `content` function for a `CBOR` node. + + +##### `entry.content({ offset, length })` + +When `entry` is a file or a `raw` node, `offset` and/or `length` arguments can be passed to `entry.content()` to return slices of data: + +```javascript +const bufs = [] + +for await (const chunk of entry.content({ + offset: 0, + length: 5 +})) { + bufs.push(chunk) +} + +// `data` contains the first 5 bytes of the file +const data = Buffer.concat(bufs) +``` + +If `entry` is a directory or hamt shard, passing `offset` and/or `length` to `entry.content()` will limit the number of files return from the directory. + +```javascript +const entries = [] + +for await (const entry of dir.content({ + offset: 0, + length: 5 +})) { + entries.push(entry) +} + +// `entries` contains the first 5 files/directories in the directory +``` + +### exporter.path(cid, ipld) + +`exporter.path` will return an async iterator that yields entries for all segments in a path: + +```javascript +const entries = [] + +for await (const entry of exporter('Qmfoo/foo/bar/baz.txt', ipld)) { + entries.push(entry) +} + +// entries contains 4x `entry` objects ``` [dag API]: https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md diff --git a/package.json b/package.json index c4e18f8..c00c0ea 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "release": "aegir release", "release-minor": "aegir release --type minor", "release-major": "aegir release --type major", - "coverage": "aegir coverage" + "coverage": "aegir coverage", + "dep-check": "aegir dep-check" }, "repository": { "type": "git", @@ -37,12 +38,17 @@ "homepage": "https://github.com/ipfs/js-ipfs-unixfs-exporter#readme", "devDependencies": { "aegir": "^18.0.2", + "async-iterator-all": "0.0.2", + "async-iterator-buffer-stream": "0.0.1", + "async-iterator-first": "0.0.2", "chai": "^4.2.0", "detect-node": "^2.0.4", "dirty-chai": "^2.0.1", - "ipld": "~0.21.1", + "ipld": "~0.22.0", "ipld-dag-pb": "~0.15.2", "ipld-in-memory": "^2.0.0", + "multicodec": "^0.5.1", + "multihashes": "^0.4.14", "pull-pushable": "^2.2.0", "pull-stream-to-stream": "^1.3.4", "pull-zip": "^2.0.1", @@ -51,14 +57,18 @@ }, "dependencies": { "async": "^2.6.1", + "async-iterator-last": "0.0.2", "cids": "~0.5.5", + "err-code": "^1.1.2", "hamt-sharding": "0.0.2", "ipfs-unixfs": "~0.1.16", - "ipfs-unixfs-importer": "~0.38.0", + "ipfs-unixfs-importer": "ipfs/js-ipfs-unixfs-importer#async-await", + "promisify-es6": "^1.0.3", "pull-cat": "^1.1.11", "pull-defer": "~0.2.3", "pull-paramap": "^1.2.2", "pull-stream": "^3.6.9", + "pull-stream-to-async-iterator": "^1.0.1", "pull-traverse": "^1.0.3" }, "contributors": [ diff --git a/src/dir-flat.js b/src/dir-flat.js deleted file mode 100644 index 53161e4..0000000 --- a/src/dir-flat.js +++ /dev/null @@ -1,53 +0,0 @@ -'use strict' - -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const filter = require('pull-stream/throughs/filter') -const map = require('pull-stream/throughs/map') -const cat = require('pull-cat') - -// Logic to export a unixfs directory. -module.exports = dirExporter - -function dirExporter (cid, node, name, path, pathRest, resolve, dag, parent, depth, options) { - const accepts = pathRest[0] - - const dir = { - name: name, - depth: depth, - path: path, - cid, - size: 0, - type: 'dir' - } - - // we are at the max depth so no need to descend into children - if (options.maxDepth && options.maxDepth <= depth) { - return values([dir]) - } - - const streams = [ - pull( - values(node.links), - filter((item) => accepts === undefined || item.name === accepts), - map((link) => ({ - depth: depth + 1, - size: 0, - name: link.name, - path: path + '/' + link.name, - cid: link.cid, - linkName: link.name, - pathRest: pathRest.slice(1), - type: 'dir' - })), - resolve - ) - ] - - // place dir before if not specifying subtree - if (!pathRest.length || options.fullPath) { - streams.unshift(values([dir])) - } - - return cat(streams) -} diff --git a/src/dir-hamt-sharded.js b/src/dir-hamt-sharded.js deleted file mode 100644 index b9ade11..0000000 --- a/src/dir-hamt-sharded.js +++ /dev/null @@ -1,203 +0,0 @@ -'use strict' - -const defer = require('pull-defer') -const pull = require('pull-stream/pull') -const error = require('pull-stream/sources/error') -const values = require('pull-stream/sources/values') -const filter = require('pull-stream/throughs/filter') -const map = require('pull-stream/throughs/map') -const cat = require('pull-cat') -const Bucket = require('hamt-sharding/src/bucket') -const DirSharded = require('ipfs-unixfs-importer/src/importer/dir-sharded') -const waterfall = require('async/waterfall') - -// Logic to export a unixfs directory. -module.exports = shardedDirExporter - -function shardedDirExporter (cid, node, name, path, pathRest, resolve, dag, parent, depth, options) { - let dir - if (!parent || (parent.path !== path)) { - dir = { - name: name, - depth: depth, - path: path, - cid, - size: 0, - type: 'dir' - } - } - - // we are at the max depth so no need to descend into children - if (options.maxDepth && options.maxDepth <= depth) { - return values([dir]) - } - - if (!pathRest.length) { - // return all children - - const streams = [ - pull( - values(node.links), - map((link) => { - // remove the link prefix (2 chars for the bucket index) - const entryName = link.name.substring(2) - const entryPath = entryName ? path + '/' + entryName : path - - return { - depth: entryName ? depth + 1 : depth, - name: entryName, - path: entryPath, - cid: link.cid, - pathRest: entryName ? pathRest.slice(1) : pathRest, - parent: dir || parent - } - }), - resolve - ) - ] - - // place dir before if not specifying subtree - streams.unshift(values([dir])) - - return cat(streams) - } - - const deferred = defer.source() - const targetFile = pathRest[0] - - // recreate our level of the HAMT so we can load only the subshard in pathRest - waterfall([ - (cb) => { - if (!options.rootBucket) { - options.rootBucket = new Bucket({ - hashFn: DirSharded.hashFn - }) - options.hamtDepth = 1 - - return addLinksToHamtBucket(node.links, options.rootBucket, options.rootBucket, cb) - } - - return addLinksToHamtBucket(node.links, options.lastBucket, options.rootBucket, cb) - }, - (cb) => findPosition(targetFile, options.rootBucket, cb), - (position, cb) => { - let prefix = toPrefix(position.pos) - const bucketPath = toBucketPath(position) - - if (bucketPath.length > (options.hamtDepth)) { - options.lastBucket = bucketPath[options.hamtDepth] - - prefix = toPrefix(options.lastBucket._posAtParent) - } - - const streams = [ - pull( - values(node.links), - map((link) => { - const entryPrefix = link.name.substring(0, 2) - const entryName = link.name.substring(2) - const entryPath = entryName ? path + '/' + entryName : path - - if (entryPrefix !== prefix) { - // not the entry or subshard we're looking for - return false - } - - if (entryName && entryName !== targetFile) { - // not the entry we're looking for - return false - } - - if (!entryName) { - // we are doing to descend into a subshard - options.hamtDepth++ - } else { - // we've found the node we are looking for, remove the context - // so we don't affect further hamt traversals - delete options.rootBucket - delete options.lastBucket - delete options.hamtDepth - } - - return { - depth: entryName ? depth + 1 : depth, - name: entryName, - path: entryPath, - cid: link.cid, - pathRest: entryName ? pathRest.slice(1) : pathRest, - parent: dir || parent - } - }), - filter(Boolean), - resolve - ) - ] - - if (options.fullPath) { - streams.unshift(values([dir])) - } - - cb(null, streams) - } - ], (err, streams) => { - if (err) { - return deferred.resolve(error(err)) - } - - deferred.resolve(cat(streams)) - }) - - return deferred -} - -const addLinksToHamtBucket = (links, bucket, rootBucket, callback) => { - Promise.all( - links.map(link => { - if (link.name.length === 2) { - const pos = parseInt(link.name, 16) - - return bucket._putObjectAt(pos, new Bucket({ - hashFn: DirSharded.hashFn - }, bucket, pos)) - } - - return rootBucket.put(link.name.substring(2), true) - }) - ) - .then(() => callback(), callback) -} - -const toPrefix = (position) => { - return position - .toString('16') - .toUpperCase() - .padStart(2, '0') - .substring(0, 2) -} - -const findPosition = (file, bucket, cb) => { - bucket._findNewBucketAndPos(file) - .then(position => { - if (!cb) { - // would have errored in catch block above - return - } - - cb(null, position) - }, cb) -} - -const toBucketPath = (position) => { - let bucket = position.bucket - const path = [] - - while (bucket._parent) { - path.push(bucket) - - bucket = bucket._parent - } - - path.push(bucket) - - return path.reverse() -} diff --git a/src/index.js b/src/index.js index 361b989..c85471b 100644 --- a/src/index.js +++ b/src/index.js @@ -1,118 +1,83 @@ 'use strict' -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const error = require('pull-stream/sources/error') -const filter = require('pull-stream/throughs/filter') -const map = require('pull-stream/throughs/map') +const errCode = require('err-code') const CID = require('cids') +const resolve = require('./resolvers') +const last = require('async-iterator-last') -const createResolver = require('./resolve').createResolver - -function pathBaseAndRest (path) { - // Buffer -> raw multihash or CID in buffer - let pathBase = path - let pathRest = '/' +const toPathComponents = (path = '') => { + // split on / unless escaped with \ + return (path + .trim() + .match(/([^\\^/]|\\\/)+/g) || []) + .filter(Boolean) +} +const cidAndRest = (path) => { if (Buffer.isBuffer(path)) { - pathBase = (new CID(path)).toBaseEncodedString() + return { + cid: new CID(path), + toResolve: [] + } + } + + if (CID.isCID(path)) { + return { + cid: path, + toResolve: [] + } } if (typeof path === 'string') { if (path.indexOf('/ipfs/') === 0) { - path = pathBase = path.substring(6) - } - const subtreeStart = path.indexOf('/') - if (subtreeStart > 0) { - pathBase = path.substring(0, subtreeStart) - pathRest = path.substring(subtreeStart) + path = path.substring(6) } - } else if (CID.isCID(pathBase)) { - pathBase = pathBase.toBaseEncodedString() - } - pathBase = (new CID(pathBase)).toBaseEncodedString() + const output = toPathComponents(path) - return { - base: pathBase, - rest: toPathComponents(pathRest) + return { + cid: new CID(output[0]), + toResolve: output.slice(1) + } } -} -const defaultOptions = { - maxDepth: Infinity, - offset: undefined, - length: undefined, - fullPath: false + throw errCode(new Error(`Unknown path type ${path}`), 'EBADPATH') } -module.exports = (path, dag, options) => { - options = Object.assign({}, defaultOptions, options) +const walkPath = async function * (path, ipld) { + let { + cid, + toResolve + } = cidAndRest(path) + let name = cid.toBaseEncodedString() + let entryPath = name - let dPath - try { - dPath = pathBaseAndRest(path) - } catch (err) { - return error(err) - } + while (true) { + const result = await resolve(cid, name, entryPath, toResolve, ipld) - const pathLengthToCut = join( - [dPath.base].concat(dPath.rest.slice(0, dPath.rest.length - 1))).length - - const cid = new CID(dPath.base) - - return pull( - values([{ - cid, - name: dPath.base, - path: dPath.base, - pathRest: dPath.rest, - depth: 0 - }]), - createResolver(dag, options), - filter(Boolean), - map((node) => { - return { - depth: node.depth, - name: node.name, - path: options.fullPath ? node.path : finalPathFor(node), - size: node.size, - cid: node.cid, - content: node.content, - type: node.type - } - }) - ) - - function finalPathFor (node) { - if (!dPath.rest.length) { - return node.path + if (!result.entry && !result.next) { + throw errCode(new Error(`Could not resolve ${path}`), 'ENOTFOUND') } - let retPath = node.path.substring(pathLengthToCut) - if (retPath.charAt(0) === '/') { - retPath = retPath.substring(1) + if (result.entry) { + yield result.entry } - if (!retPath) { - retPath = dPath.rest[dPath.rest.length - 1] || dPath.base + + if (!result.next) { + return } - return retPath + + // resolve further parts + toResolve = result.next.toResolve + cid = result.next.cid + name = result.next.name + entryPath = result.next.path } } -function join (paths) { - return paths.reduce((acc, path) => { - if (acc.length) { - acc += '/' - } - return acc + path - }, '') +const exporter = (path, ipld) => { + return last(walkPath(path, ipld)) } -const toPathComponents = (path = '') => { - // split on / unless escaped with \ - return (path - .trim() - .match(/([^\\^/]|\\\/)+/g) || []) - .filter(Boolean) -} +module.exports = exporter +module.exports.path = walkPath diff --git a/src/object.js b/src/object.js deleted file mode 100644 index eb11de1..0000000 --- a/src/object.js +++ /dev/null @@ -1,34 +0,0 @@ -'use strict' - -const CID = require('cids') -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const error = require('pull-stream/sources/error') - -module.exports = (cid, node, name, path, pathRest, resolve, dag, parent, depth) => { - let newNode - if (pathRest.length) { - const pathElem = pathRest[0] - newNode = node[pathElem] - const newName = path + '/' + pathElem - if (!newNode) { - return error(new Error(`not found`)) - } - - const isCID = CID.isCID(newNode) - - return pull( - values([{ - depth: depth, - name: pathElem, - path: newName, - pathRest: pathRest.slice(1), - cid: isCID && newNode, - object: !isCID && newNode, - parent: parent - }]), - resolve) - } else { - return error(new Error('invalid node type')) - } -} diff --git a/src/raw.js b/src/raw.js deleted file mode 100644 index 94b7462..0000000 --- a/src/raw.js +++ /dev/null @@ -1,62 +0,0 @@ -'use strict' - -const error = require('pull-stream/sources/error') -const once = require('pull-stream/sources/once') -const empty = require('pull-stream/sources/empty') -const extractDataFromBlock = require('./extract-data-from-block') - -// Logic to export a single raw block -module.exports = (cid, node, name, path, pathRest, resolve, dag, parent, depth, options) => { - const accepts = pathRest[0] - - if (accepts !== undefined && accepts !== path) { - return empty() - } - - const size = node.length - - let offset = options.offset - let length = options.length - - if (offset < 0) { - return error(new Error('Offset must be greater than or equal to 0')) - } - - if (offset > size) { - return error(new Error('Offset must be less than the file size')) - } - - if (length < 0) { - return error(new Error('Length must be greater than or equal to 0')) - } - - if (length === 0) { - return once({ - depth, - content: once(Buffer.alloc(0)), - cid, - name, - path, - size, - type: 'raw' - }) - } - - if (!offset) { - offset = 0 - } - - if (!length || (offset + length > size)) { - length = size - offset - } - - return once({ - depth, - content: once(extractDataFromBlock(node, 0, offset, offset + length)), - cid, - name, - path, - size, - type: 'raw' - }) -} diff --git a/src/resolve.js b/src/resolve.js deleted file mode 100644 index e0c10b5..0000000 --- a/src/resolve.js +++ /dev/null @@ -1,101 +0,0 @@ -'use strict' - -const UnixFS = require('ipfs-unixfs') -const pull = require('pull-stream/pull') -const error = require('pull-stream/sources/error') -const filter = require('pull-stream/throughs/filter') -const flatten = require('pull-stream/throughs/flatten') -const map = require('pull-stream/throughs/map') -const paramap = require('pull-paramap') -const waterfall = require('async/waterfall') - -const resolvers = { - directory: require('./dir-flat'), - 'hamt-sharded-directory': require('./dir-hamt-sharded'), - file: require('./file'), - object: require('./object'), - raw: require('./raw') -} - -module.exports = Object.assign({ - createResolver: createResolver, - typeOf: typeOf -}, resolvers) - -function createResolver (dag, options, depth, parent) { - if (!depth) { - depth = 0 - } - - if (depth > options.maxDepth) { - return map(identity) - } - - return pull( - paramap((item, cb) => { - if ((typeof item.depth) !== 'number') { - return error(new Error('no depth')) - } - - if (item.object) { - return cb(null, resolveItem(null, item.object, item, options)) - } - - waterfall([ - (done) => dag.get(item.cid, done), - (node, done) => done(null, resolveItem(item.cid, node.value, item, options)) - ], cb) - }), - flatten(), - filter(Boolean), - filter((node) => node.depth <= options.maxDepth) - ) - - function resolveItem (cid, node, item, options) { - return resolve({ - cid, - node, - name: item.name, - path: item.path, - pathRest: item.pathRest, - dag, - parentNode: item.parent || parent, - depth: item.depth, - options - }) - } - - function resolve ({ cid, node, name, path, pathRest, dag, parentNode, depth, options }) { - let type - - try { - type = typeOf(node) - } catch (err) { - return error(err) - } - - const nodeResolver = resolvers[type] - - if (!nodeResolver) { - return error(new Error('Unkown node type ' + type)) - } - - const resolveDeep = createResolver(dag, options, depth, node) - - return nodeResolver(cid, node, name, path, pathRest, resolveDeep, dag, parentNode, depth, options) - } -} - -function typeOf (node) { - if (Buffer.isBuffer(node)) { - return 'raw' - } else if (Buffer.isBuffer(node.data)) { - return UnixFS.unmarshal(node.data).type - } else { - return 'object' - } -} - -function identity (o) { - return o -} diff --git a/src/resolvers/dag-cbor.js b/src/resolvers/dag-cbor.js new file mode 100644 index 0000000..fcc851c --- /dev/null +++ b/src/resolvers/dag-cbor.js @@ -0,0 +1,53 @@ +'use strict' + +const CID = require('cids') +const errCode = require('err-code') + +const resolve = async (cid, name, path, toResolve, resolve, ipld) => { + let node = await ipld.get(cid) + let subObject = node + let subPath = path + + while (toResolve.length) { + const prop = toResolve[0] + + if (prop in subObject) { + // remove the bit of the path we have resolved + toResolve.shift() + subPath = `${subPath}/${prop}` + + if (CID.isCID(subObject[prop])) { + return { + entry: { + name, + path, + cid, + node + }, + next: { + cid: subObject[prop], + name: prop, + path: subPath, + toResolve + } + } + } + + subObject = subObject[prop] + } else { + // cannot resolve further + throw errCode(new Error(`No property named ${prop} found in cbor node ${cid.toBaseEncodedString()}`), 'ENOPROP') + } + } + + return { + entry: { + name, + path, + cid, + node + } + } +} + +module.exports = resolve diff --git a/src/resolvers/index.js b/src/resolvers/index.js new file mode 100644 index 0000000..a80ef7e --- /dev/null +++ b/src/resolvers/index.js @@ -0,0 +1,21 @@ +'use strict' + +const errCode = require('err-code') + +const resolvers = { + 'dag-pb': require('./unixfs-v1'), + raw: require('./raw'), + 'dag-cbor': require('./dag-cbor') +} + +const resolve = (cid, name, path, toResolve, ipld) => { + const resolver = resolvers[cid.codec] + + if (!resolver) { + throw errCode(new Error(`No resolver for codec ${cid.codec}`), 'ENORESOLVER') + } + + return resolver(cid, name, path, toResolve, resolve, ipld) +} + +module.exports = resolve diff --git a/src/resolvers/raw.js b/src/resolvers/raw.js new file mode 100644 index 0000000..ca85677 --- /dev/null +++ b/src/resolvers/raw.js @@ -0,0 +1,66 @@ +'use strict' + +const errCode = require('err-code') +const extractDataFromBlock = require('../utils/extract-data-from-block') +const toIterator = require('pull-stream-to-async-iterator') +const once = require('pull-stream/sources/once') +const error = require('pull-stream/sources/error') + +const rawContent = (node) => { + return (options = {}) => { + const size = node.length + + let offset = options.offset + let length = options.length + + if (offset < 0) { + return toIterator(error(errCode(new Error('Offset must be greater than or equal to 0'), 'EINVALIDPARAMS'))) + } + + if (offset > size) { + return toIterator(error(errCode(new Error('Offset must be less than the file size'), 'EINVALIDPARAMS'))) + } + + if (length < 0) { + return toIterator(error(errCode(new Error('Length must be greater than or equal to 0'), 'EINVALIDPARAMS'))) + } + + if (length === 0) { + return toIterator(once(Buffer.alloc(0))) + } + + if (!offset) { + offset = 0 + } + + if (!length || (offset + length > size)) { + length = size - offset + } + + return toIterator(once(extractDataFromBlock(node, 0, offset, offset + length))) + } +} + +const resolve = async (cid, name, path, toResolve, resolve, ipld) => { + const node = await ipld.get(cid) + + if (!Buffer.isBuffer(node)) { + throw errCode(new Error(`'${cid.codec}' node ${cid.toBaseEncodedString()} was not a buffer`), 'ENOBUF') + } + + if (toResolve.length) { + throw errCode(new Error(`No link named ${path} found in raw node ${cid.toBaseEncodedString()}`), 'ENOLINK') + } + + return { + entry: { + name, + path, + cid, + node, + content: rawContent(node) + } + } +} + +module.exports = resolve diff --git a/src/resolvers/unixfs-v1/content/directory.js b/src/resolvers/unixfs-v1/content/directory.js new file mode 100644 index 0000000..0e19703 --- /dev/null +++ b/src/resolvers/unixfs-v1/content/directory.js @@ -0,0 +1,17 @@ +'use strict' + +const directoryContent = (cid, node, unixfs, path, resolve, ipld) => { + return async function * (options = {}) { + const offset = options.offset || 0 + const length = options.length || node.links.length + const links = node.links.slice(offset, length) + + for (const link of links) { + const result = await resolve(link.cid, link.name, `${path}/${link.name}`, [], ipld) + + yield result.entry + } + } +} + +module.exports = directoryContent diff --git a/src/file.js b/src/resolvers/unixfs-v1/content/file.js similarity index 67% rename from src/file.js rename to src/resolvers/unixfs-v1/content/file.js index 0078a9b..1662204 100644 --- a/src/file.js +++ b/src/resolvers/unixfs-v1/content/file.js @@ -1,9 +1,10 @@ 'use strict' +const extractDataFromBlock = require('../../../utils/extract-data-from-block') +const toIterator = require('pull-stream-to-async-iterator') const traverse = require('pull-traverse') const UnixFS = require('ipfs-unixfs') const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') const error = require('pull-stream/sources/error') const once = require('pull-stream/sources/once') const empty = require('pull-stream/sources/empty') @@ -11,77 +12,31 @@ const filter = require('pull-stream/throughs/filter') const flatten = require('pull-stream/throughs/flatten') const map = require('pull-stream/throughs/map') const paramap = require('pull-paramap') -const extractDataFromBlock = require('./extract-data-from-block') +const errCode = require('err-code') -// Logic to export a single (possibly chunked) unixfs file. -module.exports = (cid, node, name, path, pathRest, resolve, dag, parent, depth, options) => { - const accepts = pathRest[0] - - if (accepts !== undefined && accepts !== path) { - return empty() +function streamBytes (ipld, node, fileSize, { offset, length }) { + if (offset === fileSize || length === 0) { + return once(Buffer.alloc(0)) } - let file - - try { - file = UnixFS.unmarshal(node.data) - } catch (err) { - return error(err) + if (!offset) { + offset = 0 } - const fileSize = file.fileSize() - - let offset = options.offset - let length = options.length + if (!length) { + length = fileSize + } if (offset < 0) { - return error(new Error('Offset must be greater than or equal to 0')) + return error(errCode(new Error('Offset must be greater than or equal to 0'), 'EINVALIDPARAMS')) } if (offset > fileSize) { - return error(new Error('Offset must be less than the file size')) + return error(errCode(new Error('Offset must be less than the file size'), 'EINVALIDPARAMS')) } if (length < 0) { - return error(new Error('Length must be greater than or equal to 0')) - } - - if (length === 0) { - return once({ - depth: depth, - content: once(Buffer.alloc(0)), - name: name, - path: path, - cid, - size: fileSize, - type: 'file' - }) - } - - if (!offset) { - offset = 0 - } - - if (!length || (offset + length > fileSize)) { - length = fileSize - offset - } - - const content = streamBytes(dag, node, fileSize, offset, length) - - return values([{ - depth: depth, - content: content, - name: name, - path: path, - cid, - size: fileSize, - type: 'file' - }]) -} - -function streamBytes (dag, node, fileSize, offset, length) { - if (offset === fileSize || length === 0) { - return once(Buffer.alloc(0)) + return error(errCode(new Error('Length must be greater than or equal to 0'), 'EINVALIDPARAMS')) } const end = offset + length @@ -91,7 +46,7 @@ function streamBytes (dag, node, fileSize, offset, length) { node, start: 0, end: fileSize - }, getChildren(dag, offset, end)), + }, getChildren(ipld, offset, end)), map(extractData(offset, end)), filter(Boolean) ) @@ -150,23 +105,25 @@ function getChildren (dag, offset, end) { return pull( once(filteredLinks), - paramap((children, cb) => { - dag.getMany(children.map(child => child.link.cid), (err, results) => { - if (err) { - return cb(err) - } + paramap(async (children, cb) => { + try { + let results = [] - cb(null, results.map((result, index) => { - const child = children[index] + for await (const result of await dag.getMany(children.map(child => child.link.cid))) { + const child = children[results.length] - return { + results.push({ start: child.start, end: child.end, node: result, size: child.size - } - })) - }) + }) + } + + cb(null, results) + } catch (err) { + cb(err) + } }), flatten() ) @@ -214,3 +171,11 @@ function extractData (requestedStart, requestedEnd) { return Buffer.alloc(0) } } + +const fileContent = (cid, node, unixfs, path, resolve, ipld) => { + return (options = {}) => { + return toIterator(streamBytes(ipld, node, unixfs.fileSize(), options)) + } +} + +module.exports = fileContent diff --git a/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js b/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js new file mode 100644 index 0000000..1c9d3ab --- /dev/null +++ b/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js @@ -0,0 +1,26 @@ +'use strict' + +const hamtShardedDirectoryContent = (cid, node, unixfs, path, resolve, ipld) => { + return async function * (options = {}) { + const links = node.links + + for (const link of links) { + const name = link.name.substring(2) + + if (name) { + const result = await resolve(link.cid, name, `${path}/${name}`, [], ipld) + + yield result.entry + } else { + // descend into subshard + node = await ipld.get(link.cid) + + for await (const file of hamtShardedDirectoryContent(link.cid, node, null, path, resolve, ipld)(options)) { + yield file + } + } + } + } +} + +module.exports = hamtShardedDirectoryContent diff --git a/src/resolvers/unixfs-v1/content/raw.js b/src/resolvers/unixfs-v1/content/raw.js new file mode 100644 index 0000000..7e905de --- /dev/null +++ b/src/resolvers/unixfs-v1/content/raw.js @@ -0,0 +1,43 @@ +'use strict' + +const errCode = require('err-code') +const extractDataFromBlock = require('../../../utils/extract-data-from-block') +const toIterator = require('pull-stream-to-async-iterator') +const once = require('pull-stream/sources/once') + +const rawContent = async (cid, node, unixfs, path, resolve, ipld) => { + return (options = {}) => { + const size = node.length + + let offset = options.offset + let length = options.length + + if (offset < 0) { + throw errCode(new Error('Offset must be greater than or equal to 0'), 'EINVALIDPARAMS') + } + + if (offset > size) { + throw errCode(new Error('Offset must be less than the file size'), 'EINVALIDPARAMS') + } + + if (length < 0) { + throw errCode(new Error('Length must be greater than or equal to 0'), 'EINVALIDPARAMS') + } + + if (length === 0) { + return toIterator(once(Buffer.alloc(0))) + } + + if (!offset) { + offset = 0 + } + + if (!length || (offset + length > size)) { + length = size - offset + } + + return toIterator(once(extractDataFromBlock(unixfs.data, 0, offset, offset + length))) + } +} + +module.exports = rawContent diff --git a/src/resolvers/unixfs-v1/index.js b/src/resolvers/unixfs-v1/index.js new file mode 100644 index 0000000..2d356f4 --- /dev/null +++ b/src/resolvers/unixfs-v1/index.js @@ -0,0 +1,81 @@ +'use strict' + +const errCode = require('err-code') +const UnixFS = require('ipfs-unixfs') +const findShardCid = require('../../utils/find-cid-in-shard') + +const findLinkCid = (node, name) => { + const link = node.links.find(link => link.name === name) + + return link && link.cid +} + +const contentExporters = { + raw: require('./content/file'), + file: require('./content/file'), + directory: require('./content/directory'), + 'hamt-sharded-directory': require('./content/hamt-sharded-directory'), + metadata: (cid, node, unixfs, ipld) => {}, + symlink: (cid, node, unixfs, ipld) => {} +} + +const unixFsResolver = async (cid, name, path, toResolve, resolve, ipld) => { + const node = await ipld.get(cid) + let unixfs + let next + + if (!name) { + name = cid.toBaseEncodedString() + } + + try { + unixfs = UnixFS.unmarshal(node.data) + } catch (err) { + // non-UnixFS dag-pb node? It could happen. + throw err + } + + if (!path) { + path = name + } + + if (toResolve.length) { + let linkCid + + if (unixfs && unixfs.type === 'hamt-sharded-directory') { + // special case - unixfs v1 hamt shards + linkCid = await findShardCid(node, toResolve[0], ipld) + } else { + linkCid = findLinkCid(node, toResolve[0]) + } + + if (!linkCid) { + throw errCode(new Error(`No link named ${toResolve} found in node ${cid.toBaseEncodedString()}`), 'ENOLINK') + } + + // remove the path component we have resolved + const nextName = toResolve.shift() + const nextPath = `${path}/${nextName}` + + next = { + cid: linkCid, + toResolve, + name: nextName, + path: nextPath + } + } + + return { + entry: { + name, + path, + cid, + node, + content: contentExporters[unixfs.type](cid, node, unixfs, path, resolve, ipld), + unixfs + }, + next + } +} + +module.exports = unixFsResolver diff --git a/src/extract-data-from-block.js b/src/utils/extract-data-from-block.js similarity index 100% rename from src/extract-data-from-block.js rename to src/utils/extract-data-from-block.js diff --git a/src/utils/find-cid-in-shard.js b/src/utils/find-cid-in-shard.js new file mode 100644 index 0000000..789b5b8 --- /dev/null +++ b/src/utils/find-cid-in-shard.js @@ -0,0 +1,101 @@ +'use strict' + +const Bucket = require('hamt-sharding/src/bucket') +const DirSharded = require('ipfs-unixfs-importer/src/importer/dir-sharded') + +const addLinksToHamtBucket = (links, bucket, rootBucket) => { + return Promise.all( + links.map(link => { + if (link.name.length === 2) { + const pos = parseInt(link.name, 16) + + return bucket._putObjectAt(pos, new Bucket({ + hashFn: DirSharded.hashFn + }, bucket, pos)) + } + + return rootBucket.put(link.name.substring(2), true) + }) + ) +} + +const toPrefix = (position) => { + return position + .toString('16') + .toUpperCase() + .padStart(2, '0') + .substring(0, 2) +} + +const toBucketPath = (position) => { + let bucket = position.bucket + const path = [] + + while (bucket._parent) { + path.push(bucket) + + bucket = bucket._parent + } + + path.push(bucket) + + return path.reverse() +} + +const findShardCid = async (node, name, ipld, context) => { + if (!context) { + context = { + rootBucket: new Bucket({ + hashFn: DirSharded.hashFn + }), + hamtDepth: 1 + } + + context.lastBucket = context.rootBucket + } + + await addLinksToHamtBucket(node.links, context.lastBucket, context.rootBucket) + + const position = await context.rootBucket._findNewBucketAndPos(name) + let prefix = toPrefix(position.pos) + const bucketPath = toBucketPath(position) + + if (bucketPath.length > (context.hamtDepth)) { + context.lastBucket = bucketPath[context.hamtDepth] + + prefix = toPrefix(context.lastBucket._posAtParent) + } + + const link = node.links.find(link => { + const entryPrefix = link.name.substring(0, 2) + const entryName = link.name.substring(2) + + if (entryPrefix !== prefix) { + // not the entry or subshard we're looking for + return + } + + if (entryName && entryName !== name) { + // not the entry we're looking for + return + } + + return true + }) + + if (!link) { + return null + } + + if (link.name.substring(2) === name) { + return link.cid + } + + context.hamtDepth++ + + node = await ipld.get(link.cid) + + return findShardCid(node, name, ipld, context) +} + +module.exports = findShardCid diff --git a/test/exporter-sharded.spec.js b/test/exporter-sharded.spec.js index 57bb18b..e6bca31 100644 --- a/test/exporter-sharded.spec.js +++ b/test/exporter-sharded.spec.js @@ -7,19 +7,17 @@ const expect = chai.expect const IPLD = require('ipld') const inMemory = require('ipld-in-memory') const UnixFS = require('ipfs-unixfs') -const pull = require('pull-stream/pull') -const values = require('pull-stream/sources/values') -const collect = require('pull-stream/sinks/collect') -const CID = require('cids') -const waterfall = require('async/waterfall') -const parallel = require('async/parallel') -const randomBytes = require('./helpers/random-bytes') +const mh = require('multihashes') +const mc = require('multicodec') +const all = require('async-iterator-all') +const last = require('async-iterator-last') +const randomBytes = require('async-iterator-buffer-stream') const exporter = require('../src') const importer = require('ipfs-unixfs-importer') const { DAGLink, DAGNode -} = require('ipld-dag-pb') +} = require('./helpers/dag-pb') const SHARD_SPLIT_THRESHOLD = 10 @@ -28,30 +26,24 @@ describe('exporter sharded', function () { let ipld - const createShard = (numFiles, callback) => { - createShardWithFileNames(numFiles, (index) => `file-${index}`, callback) + const createShard = (numFiles) => { + return createShardWithFileNames(numFiles, (index) => `file-${index}`) } - const createShardWithFileNames = (numFiles, fileName, callback) => { + const createShardWithFileNames = (numFiles, fileName) => { const files = new Array(numFiles).fill(0).map((_, index) => ({ path: fileName(index), content: Buffer.from([0, 1, 2, 3, 4, index]) })) - createShardWithFiles(files, callback) + return createShardWithFiles(files) } - const createShardWithFiles = (files, callback) => { - pull( - values(files), - importer(ipld, { - shardSplitThreshold: SHARD_SPLIT_THRESHOLD, - wrap: true - }), - collect((err, files) => { - callback(err, files ? new CID(files.pop().multihash) : undefined) - }) - ) + const createShardWithFiles = async (files) => { + return (await last(importer(files, ipld, { + shardSplitThreshold: SHARD_SPLIT_THRESHOLD, + wrap: true + }))).cid } before((done) => { @@ -64,350 +56,158 @@ describe('exporter sharded', function () { }) }) - it('exports a sharded directory', (done) => { + it('exports a sharded directory', async () => { const files = {} - let directory for (let i = 0; i < (SHARD_SPLIT_THRESHOLD + 1); i++) { files[`file-${Math.random()}.txt`] = { - content: randomBytes(100) + content: Buffer.concat(await all(randomBytes(100))) } } - waterfall([ - (cb) => pull( - pull.values( - Object.keys(files).map(path => ({ - path, - content: files[path].content - })) - ), - importer(ipld, { - wrap: true, - shardSplitThreshold: SHARD_SPLIT_THRESHOLD - }), - collect(cb) - ), - (imported, cb) => { - directory = new CID(imported.pop().multihash) - - // store the CIDs, we will validate them later - imported.forEach(imported => { - files[imported.path].cid = new CID(imported.multihash) - }) - - ipld.get(directory, cb) - }, - ({ value, cid }, cb) => { - const dir = UnixFS.unmarshal(value.data) - - expect(dir.type).to.equal('hamt-sharded-directory') - - pull( - exporter(directory, ipld), - collect(cb) - ) - }, - (exported, cb) => { - const dir = exported.shift() - - expect(dir.cid.equals(directory)).to.be.true() - expect(exported.length).to.equal(Object.keys(files).length) - - parallel( - exported.map(exported => (cb) => { - pull( - exported.content, - collect((err, bufs) => { - if (err) { - cb(err) - } - - // validate the CID - expect(files[exported.name].cid.equals(exported.cid)).to.be.true() - - // validate the exported file content - expect(files[exported.name].content).to.deep.equal(bufs[0]) - - cb() - }) - ) - }), - cb - ) - } - ], done) - }) + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: files[path].content + })), ipld, { + wrap: true, + shardSplitThreshold: SHARD_SPLIT_THRESHOLD + })) - it('exports all the files from a sharded directory with maxDepth', (done) => { - const files = {} - let dirCid + const dirCid = imported.pop().cid - for (let i = 0; i < (SHARD_SPLIT_THRESHOLD + 1); i++) { - files[`file-${Math.random()}.txt`] = { - content: randomBytes(100) - } - } + // store the CIDs, we will validate them later + imported.forEach(imported => { + files[imported.path].cid = imported.cid + }) - waterfall([ - (cb) => pull( - pull.values( - Object.keys(files).map(path => ({ - path, - content: files[path].content - })) - ), - importer(ipld, { - wrap: true, - shardSplitThreshold: SHARD_SPLIT_THRESHOLD - }), - collect(cb) - ), - (imported, cb) => { - dirCid = new CID(imported.pop().multihash) - - pull( - exporter(dirCid, ipld, { - maxDepth: 1 - }), - collect(cb) - ) - }, - (exported, cb) => { - const dir = exported.shift() - - expect(dir.cid.equals(dirCid)).to.be.true() - expect(exported.length).to.equal(Object.keys(files).length) - - cb() - } - ], done) - }) + const dir = await ipld.get(dirCid) + const dirMetadata = UnixFS.unmarshal(dir.data) - it('exports all files from a sharded directory with subshards', (done) => { - waterfall([ - (cb) => createShard(31, cb), - (dir, cb) => { - pull( - exporter(`/ipfs/${dir.toBaseEncodedString()}`, ipld), - collect(cb) - ) - }, - (exported, cb) => { - expect(exported.length).to.equal(32) + expect(dirMetadata.type).to.equal('hamt-sharded-directory') - const dir = exported.shift() + const exported = await exporter(dirCid, ipld) - expect(dir.type).to.equal('dir') + expect(exported.cid.equals(dirCid)).to.be.true() - exported.forEach(file => expect(file.type).to.equal('file')) + const dirFiles = await all(exported.content()) + expect(dirFiles.length).to.equal(Object.keys(files).length) - cb() - } - ], done) - }) + for (let i = 0; i < dirFiles.length; i++) { + const dirFile = dirFiles[i] + const data = Buffer.concat(await all(dirFile.content())) - it('exports one file from a sharded directory', (done) => { - waterfall([ - (cb) => createShard(31, cb), - (dir, cb) => { - pull( - exporter(`/ipfs/${dir.toBaseEncodedString()}/file-14`, ipld), - collect(cb) - ) - }, - (exported, cb) => { - expect(exported.length).to.equal(1) + // validate the CID + expect(files[dirFile.name].cid.equals(dirFile.cid)).to.be.true() + + // validate the exported file content + expect(files[dirFile.name].content).to.deep.equal(data) + } + }) - const file = exported.shift() + it('exports all files from a sharded directory with subshards', async () => { + const numFiles = 31 + const dirCid = await createShard(numFiles) + const exported = await exporter(dirCid, ipld) + const files = await all(exported.content()) + expect(files.length).to.equal(numFiles) - expect(file.name).to.deep.equal('file-14') + expect(exported.unixfs.type).to.equal('hamt-sharded-directory') - cb() - } - ], done) + files.forEach(file => expect(file.unixfs.type).to.equal('file')) }) - it('exports one file from a sharded directory sub shard', (done) => { - waterfall([ - (cb) => createShard(31, cb), - (dir, cb) => { - pull( - exporter(`/ipfs/${dir.toBaseEncodedString()}/file-30`, ipld), - collect(cb) - ) - }, - (exported, cb) => { - expect(exported.length).to.equal(1) + it('exports one file from a sharded directory', async () => { + const dirCid = await createShard(31) + const exported = await exporter(`/ipfs/${dirCid.toBaseEncodedString()}/file-14`, ipld) - const file = exported.shift() + expect(exported.name).to.equal('file-14') + }) - expect(file.name).to.deep.equal('file-30') + it('exports one file from a sharded directory sub shard', async () => { + const dirCid = await createShard(31) + const exported = await exporter(`/ipfs/${dirCid.toBaseEncodedString()}/file-30`, ipld) - cb() - } - ], done) + expect(exported.name).to.deep.equal('file-30') }) - it('exports one file from a shard inside a shard inside a shard', (done) => { - waterfall([ - (cb) => createShard(2568, cb), - (dir, cb) => { - pull( - exporter(`/ipfs/${dir.toBaseEncodedString()}/file-2567`, ipld), - collect(cb) - ) - }, - (exported, cb) => { - expect(exported.length).to.equal(1) + it('exports one file from a shard inside a shard inside a shard', async () => { + const dirCid = await createShard(2568) + const exported = await exporter(`/ipfs/${dirCid.toBaseEncodedString()}/file-2567`, ipld) - const file = exported.shift() + expect(exported.name).to.deep.equal('file-2567') + }) - expect(file.name).to.deep.equal('file-2567') + it('extracts a deep folder from the sharded directory', async () => { + const dirCid = await createShardWithFileNames(31, (index) => `/foo/bar/baz/file-${index}`) + const exported = await exporter(`/ipfs/${dirCid.toBaseEncodedString()}/foo/bar/baz`, ipld) - cb() - } - ], done) + expect(exported.name).to.deep.equal('baz') }) - it('uses maxDepth to only extract a deep folder from the sharded directory', (done) => { - waterfall([ - (cb) => createShardWithFileNames(31, (index) => `/foo/bar/baz/file-${index}`, cb), - (dir, cb) => { - pull( - exporter(`/ipfs/${dir.toBaseEncodedString()}/foo/bar/baz`, ipld, { - maxDepth: 3 - }), - collect(cb) - ) - }, - (exported, cb) => { - expect(exported.length).to.equal(1) - - const entry = exported.pop() - - expect(entry.name).to.deep.equal('baz') - - cb() - } - ], done) - }) + it('extracts an intermediate folder from the sharded directory', async () => { + const dirCid = await createShardWithFileNames(31, (index) => `/foo/bar/baz/file-${index}`) + const exported = await exporter(`/ipfs/${dirCid.toBaseEncodedString()}/foo/bar`, ipld) - it('uses maxDepth to only extract an intermediate folder from the sharded directory', (done) => { - waterfall([ - (cb) => createShardWithFileNames(31, (index) => `/foo/bar/baz/file-${index}`, cb), - (dir, cb) => { - pull( - exporter(`/ipfs/${dir.toBaseEncodedString()}/foo/bar/baz`, ipld, { - maxDepth: 2 - }), - collect(cb) - ) - }, - (exported, cb) => { - expect(exported.length).to.equal(1) - - const entry = exported.pop() - - expect(entry.name).to.deep.equal('bar') - - cb() - } - ], done) + expect(exported.name).to.deep.equal('bar') }) - it('uses fullPath extract all intermediate entries from the sharded directory', (done) => { - waterfall([ - (cb) => createShardWithFileNames(31, (index) => `/foo/bar/baz/file-${index}`, cb), - (dir, cb) => { - pull( - exporter(`/ipfs/${dir.toBaseEncodedString()}/foo/bar/baz/file-1`, ipld, { - fullPath: true - }), - collect(cb) - ) - }, - (exported, cb) => { - expect(exported.length).to.equal(5) - - expect(exported[1].name).to.equal('foo') - expect(exported[2].name).to.equal('bar') - expect(exported[3].name).to.equal('baz') - expect(exported[4].name).to.equal('file-1') - - cb() - } - ], done) + it('uses .path to extract all intermediate entries from the sharded directory', async () => { + const dirCid = await createShardWithFileNames(31, (index) => `/foo/bar/baz/file-${index}`) + const exported = await all(exporter.path(`/ipfs/${dirCid.toBaseEncodedString()}/foo/bar/baz/file-1`, ipld)) + + expect(exported.length).to.equal(5) + + expect(exported[0].name).to.equal(dirCid.toBaseEncodedString()) + expect(exported[1].name).to.equal('foo') + expect(exported[1].path).to.equal(`${dirCid.toBaseEncodedString()}/foo`) + expect(exported[2].name).to.equal('bar') + expect(exported[2].path).to.equal(`${dirCid.toBaseEncodedString()}/foo/bar`) + expect(exported[3].name).to.equal('baz') + expect(exported[3].path).to.equal(`${dirCid.toBaseEncodedString()}/foo/bar/baz`) + expect(exported[4].name).to.equal('file-1') + expect(exported[4].path).to.equal(`${dirCid.toBaseEncodedString()}/foo/bar/baz/file-1`) }) - it('uses fullPath extract all intermediate entries from the sharded directory as well as the contents', (done) => { - waterfall([ - (cb) => createShardWithFileNames(31, (index) => `/foo/bar/baz/file-${index}`, cb), - (dir, cb) => { - pull( - exporter(`/ipfs/${dir.toBaseEncodedString()}/foo/bar/baz`, ipld, { - fullPath: true - }), - collect(cb) - ) - }, - (exported, cb) => { - expect(exported.length).to.equal(35) - - expect(exported[1].name).to.equal('foo') - expect(exported[2].name).to.equal('bar') - expect(exported[3].name).to.equal('baz') - expect(exported[4].name).to.equal('file-14') - - exported.slice(4).forEach(file => expect(file.type).to.equal('file')) - - cb() - } - ], done) + it('uses .path to extract all intermediate entries from the sharded directory as well as the contents', async () => { + const dirCid = await createShardWithFileNames(31, (index) => `/foo/bar/baz/file-${index}`) + const exported = await all(exporter.path(`/ipfs/${dirCid.toBaseEncodedString()}/foo/bar/baz`, ipld)) + + expect(exported.length).to.equal(4) + + expect(exported[1].name).to.equal('foo') + expect(exported[2].name).to.equal('bar') + expect(exported[3].name).to.equal('baz') + + const files = await all(exported[3].content()) + + expect(files.length).to.equal(31) + + files.forEach(file => { + expect(file.unixfs.type).to.equal('file') + }) }) - it('exports a file from a sharded directory inside a regular directory inside a sharded directory', (done) => { - waterfall([ - (cb) => createShard(15, cb), - (dir, cb) => { - DAGNode.create(new UnixFS('directory').marshal(), [ - new DAGLink('shard', 5, dir) - ], cb) - }, - (node, cb) => { - ipld.put(node, { - version: 0, - format: 'dag-pb', - hashAlg: 'sha2-256' - }, cb) - }, - (cid, cb) => { - DAGNode.create(new UnixFS('hamt-sharded-directory').marshal(), [ - new DAGLink('75normal-dir', 5, cid) - ], cb) - }, - (node, cb) => { - ipld.put(node, { - version: 1, - format: 'dag-pb', - hashAlg: 'sha2-256' - }, cb) - }, - (dir, cb) => { - pull( - exporter(`/ipfs/${dir.toBaseEncodedString()}/normal-dir/shard/file-1`, ipld), - collect(cb) - ) - }, - (exported, cb) => { - expect(exported.length).to.equal(1) - - const entry = exported.pop() - - expect(entry.name).to.deep.equal('file-1') - - cb() - } - ], done) + it('exports a file from a sharded directory inside a regular directory inside a sharded directory', async () => { + const dirCid = await createShard(15) + + const node = await DAGNode.create(new UnixFS('directory').marshal(), [ + await DAGLink.create('shard', 5, dirCid) + ]) + const nodeCid = await ipld.put(node, mc.DAG_PB, { + cidVersion: 0, + hashAlg: mh.names['sha2-256'] + }) + + const shardNode = await DAGNode.create(new UnixFS('hamt-sharded-directory').marshal(), [ + await DAGLink.create('75normal-dir', 5, nodeCid) + ]) + const shardNodeCid = await ipld.put(shardNode, mc.DAG_PB, { + cidVersion: 1, + hashAlg: mh.names['sha2-256'] + }) + + const exported = await exporter(`/ipfs/${shardNodeCid.toBaseEncodedString()}/normal-dir/shard/file-1`, ipld) + + expect(exported.name).to.deep.equal('file-1') }) }) diff --git a/test/exporter-subtree.spec.js b/test/exporter-subtree.spec.js index a4577ef..a7f66cb 100644 --- a/test/exporter-subtree.spec.js +++ b/test/exporter-subtree.spec.js @@ -6,11 +6,11 @@ chai.use(require('dirty-chai')) const expect = chai.expect const IPLD = require('ipld') const inMemory = require('ipld-in-memory') -const CID = require('cids') -const pull = require('pull-stream') -const randomBytes = require('./helpers/random-bytes') -const waterfall = require('async/waterfall') const importer = require('ipfs-unixfs-importer') +const mc = require('multicodec') +const all = require('async-iterator-all') +const last = require('async-iterator-last') +const randomBytes = require('async-iterator-buffer-stream') const ONE_MEG = Math.pow(1024, 2) @@ -29,185 +29,117 @@ describe('exporter subtree', () => { }) }) - it('exports a file 2 levels down', (done) => { - const content = randomBytes(ONE_MEG) - - waterfall([ - (cb) => pull( - pull.values([{ - path: './200Bytes.txt', - content: randomBytes(ONE_MEG) - }, { - path: './level-1/200Bytes.txt', - content - }]), - importer(ipld), - pull.collect(cb) - ), - (files, cb) => cb(null, files.pop().multihash), - (buf, cb) => cb(null, new CID(buf)), - (cid, cb) => pull( - exporter(`${cid.toBaseEncodedString()}/level-1/200Bytes.txt`, ipld), - pull.collect((err, files) => cb(err, { cid, files })) - ), - ({ cid, files }, cb) => { - files.forEach(file => expect(file).to.have.property('cid')) - - expect(files.length).to.equal(1) - expect(files[0].path).to.equal('200Bytes.txt') - fileEql(files[0], content, cb) - } - ], done) + it('exports a file 2 levels down', async () => { + const content = Buffer.concat(await all(randomBytes(ONE_MEG))) + + const imported = await last(importer([{ + path: './200Bytes.txt', + content: randomBytes(ONE_MEG) + }, { + path: './level-1/200Bytes.txt', + content + }], ipld)) + + const exported = await exporter(`${imported.cid.toBaseEncodedString()}/level-1/200Bytes.txt`, ipld) + + expect(exported).to.have.property('cid') + expect(exported.name).to.equal('200Bytes.txt') + expect(exported.path).to.equal(`${imported.cid.toBaseEncodedString()}/level-1/200Bytes.txt`) + + const data = Buffer.concat(await all(exported.content())) + expect(data).to.deep.equal(content) }) - it('exports a directory 1 level down', (done) => { - const content = randomBytes(ONE_MEG) - - waterfall([ - (cb) => pull( - pull.values([{ - path: './200Bytes.txt', - content: randomBytes(ONE_MEG) - }, { - path: './level-1/200Bytes.txt', - content - }, { - path: './level-1/level-2' - }]), - importer(ipld), - pull.collect(cb) - ), - (files, cb) => cb(null, files.pop().multihash), - (buf, cb) => cb(null, new CID(buf)), - (cid, cb) => pull( - exporter(`${cid.toBaseEncodedString()}/level-1`, ipld), - pull.collect((err, files) => cb(err, { cid, files })) - ), - ({ cid, files }, cb) => { - expect(files.length).to.equal(3) - expect(files[0].path).to.equal('level-1') - expect(files[1].path).to.equal('level-1/200Bytes.txt') - expect(files[2].path).to.equal('level-1/level-2') - fileEql(files[1], content, cb) - } - ], done) + it('exports a directory 1 level down', async () => { + const content = Buffer.concat(await all(randomBytes(ONE_MEG))) + const imported = await last(importer([{ + path: './200Bytes.txt', + content: randomBytes(ONE_MEG) + }, { + path: './level-1/200Bytes.txt', + content + }, { + path: './level-1/level-2' + }], ipld)) + + const exported = await exporter(`${imported.cid.toBaseEncodedString()}/level-1`, ipld) + const files = await all(exported.content()) + + expect(files.length).to.equal(2) + expect(files[0].name).to.equal('200Bytes.txt') + expect(files[0].path).to.equal(`${imported.cid.toBaseEncodedString()}/level-1/200Bytes.txt`) + + expect(files[1].name).to.equal('level-2') + expect(files[1].path).to.equal(`${imported.cid.toBaseEncodedString()}/level-1/level-2`) + + const data = Buffer.concat(await all(files[0].content())) + expect(data).to.deep.equal(content) }) - it('export a non existing file from a directory', (done) => { - waterfall([ - (cb) => pull( - pull.values([{ - path: '/derp/200Bytes.txt', - content: randomBytes(ONE_MEG) - }]), - importer(ipld), - pull.collect(cb) - ), - (files, cb) => cb(null, files.pop().multihash), - (buf, cb) => cb(null, new CID(buf)), - (cid, cb) => pull( - exporter(`${cid.toBaseEncodedString()}/doesnotexist`, ipld), - pull.collect((err, files) => cb(err, { cid, files })) - ), - ({ cid, files }, cb) => { - expect(files.length).to.equal(0) - cb() - } - ], done) + it('exports a non existing file from a directory', async () => { + const imported = await last(importer([{ + path: '/derp/200Bytes.txt', + content: randomBytes(ONE_MEG) + }], ipld)) + + try { + await exporter(`${imported.cid.toBaseEncodedString()}/doesnotexist`, ipld) + } catch (err) { + expect(err.code).to.equal('ENOLINK') + } }) - it('exports starting from non-protobuf node', (done) => { - const content = randomBytes(ONE_MEG) - - waterfall([ - (cb) => pull( - pull.values([{ - path: './level-1/200Bytes.txt', - content - }]), - importer(ipld, { - wrapWithDirectory: true - }), - pull.collect(cb) - ), - (files, cb) => cb(null, files.pop().multihash), - (buf, cb) => cb(null, new CID(buf)), - (cid, cb) => ipld.put({ a: { file: cid } }, { format: 'dag-cbor' }, cb), - (cborNodeCid, cb) => pull( - exporter(`${cborNodeCid.toBaseEncodedString()}/a/file/level-1/200Bytes.txt`, ipld), - pull.collect(cb) - ), - (files, cb) => { - expect(files.length).to.equal(1) - expect(files[0].path).to.equal('200Bytes.txt') - fileEql(files[0], content, cb) + it('exports starting from non-protobuf node', async () => { + const content = Buffer.concat(await all(randomBytes(ONE_MEG))) + + const imported = await last(importer([{ + path: './level-1/200Bytes.txt', + content + }], ipld, { + wrapWithDirectory: true + })) + + const cborNodeCid = await ipld.put({ + a: { + file: imported.cid } - ], done) + }, mc.DAG_CBOR) + + const exported = await exporter(`${cborNodeCid.toBaseEncodedString()}/a/file/level-1/200Bytes.txt`, ipld) + + expect(exported.name).to.equal('200Bytes.txt') + expect(exported.path).to.equal(`${cborNodeCid.toBaseEncodedString()}/a/file/level-1/200Bytes.txt`) + + const data = Buffer.concat(await all(exported.content())) + expect(data).to.deep.equal(content) }) - it('exports all components of a path', (done) => { - const content = randomBytes(ONE_MEG) - - waterfall([ - (cb) => pull( - pull.values([{ - path: './200Bytes.txt', - content: randomBytes(ONE_MEG) - }, { - path: './level-1/200Bytes.txt', - content - }, { - path: './level-1/level-2' - }, { - path: './level-1/level-2/200Bytes.txt', - content - }]), - importer(ipld), - pull.collect(cb) - ), - (files, cb) => cb(null, files.pop().multihash), - (buf, cb) => cb(null, new CID(buf)), - (cid, cb) => pull( - exporter(`${cid.toBaseEncodedString()}/level-1/level-2/200Bytes.txt`, ipld, { - fullPath: true - }), - pull.collect((err, files) => cb(err, { cid, files })) - ), - ({ cid, files }, cb) => { - expect(files.length).to.equal(4) - expect(files[0].path).to.equal(cid.toBaseEncodedString()) - expect(files[0].name).to.equal(cid.toBaseEncodedString()) - expect(files[1].path).to.equal(`${cid.toBaseEncodedString()}/level-1`) - expect(files[1].name).to.equal('level-1') - expect(files[2].path).to.equal(`${cid.toBaseEncodedString()}/level-1/level-2`) - expect(files[2].name).to.equal('level-2') - expect(files[3].path).to.equal(`${cid.toBaseEncodedString()}/level-1/level-2/200Bytes.txt`) - expect(files[3].name).to.equal('200Bytes.txt') - - cb() - } - ], done) + it('uses .path to export all components of a path', async () => { + const content = Buffer.concat(await all(randomBytes(ONE_MEG))) + + const imported = await last(importer([{ + path: './200Bytes.txt', + content: randomBytes(ONE_MEG) + }, { + path: './level-1/200Bytes.txt', + content + }, { + path: './level-1/level-2' + }, { + path: './level-1/level-2/200Bytes.txt', + content + }], ipld)) + + const exported = await all(exporter.path(`${imported.cid.toBaseEncodedString()}/level-1/level-2/200Bytes.txt`, ipld)) + + expect(exported.length).to.equal(4) + expect(exported[0].path).to.equal(imported.cid.toBaseEncodedString()) + expect(exported[0].name).to.equal(imported.cid.toBaseEncodedString()) + expect(exported[1].path).to.equal(`${imported.cid.toBaseEncodedString()}/level-1`) + expect(exported[1].name).to.equal('level-1') + expect(exported[2].path).to.equal(`${imported.cid.toBaseEncodedString()}/level-1/level-2`) + expect(exported[2].name).to.equal('level-2') + expect(exported[3].path).to.equal(`${imported.cid.toBaseEncodedString()}/level-1/level-2/200Bytes.txt`) + expect(exported[3].name).to.equal('200Bytes.txt') }) }) - -function fileEql (f1, f2, done) { - pull( - f1.content, - pull.collect((err, data) => { - if (err) { - return done(err) - } - - try { - if (f2) { - expect(Buffer.concat(data)).to.eql(f2) - } else { - expect(data).to.exist() - } - } catch (err) { - return done(err) - } - done() - }) - ) -} diff --git a/test/exporter.spec.js b/test/exporter.spec.js index 124b709..df57f4a 100644 --- a/test/exporter.spec.js +++ b/test/exporter.spec.js @@ -7,147 +7,73 @@ const expect = chai.expect const IPLD = require('ipld') const inMemory = require('ipld-in-memory') const UnixFS = require('ipfs-unixfs') -const pull = require('pull-stream') -const zip = require('pull-zip') const CID = require('cids') -const doUntil = require('async/doUntil') -const waterfall = require('async/waterfall') -const parallel = require('async/parallel') -const series = require('async/series') -const fs = require('fs') -const path = require('path') -const push = require('pull-pushable') -const toPull = require('stream-to-pull-stream') -const toStream = require('pull-stream-to-stream') const { DAGNode, DAGLink -} = require('ipld-dag-pb') -const isNode = require('detect-node') -const randomBytes = require('./helpers/random-bytes') - +} = require('./helpers/dag-pb') +const mh = require('multihashes') +const mc = require('multicodec') const exporter = require('../src') const importer = require('ipfs-unixfs-importer') +const all = require('async-iterator-all') +const last = require('async-iterator-last') +const first = require('async-iterator-first') +const randomBytes = require('async-iterator-buffer-stream') const ONE_MEG = Math.pow(1024, 2) -const bigFile = randomBytes(ONE_MEG * 1.2) -const smallFile = randomBytes(200) describe('exporter', () => { let ipld + let bigFile + let smallFile - function dagPut (options, cb) { - if (typeof options === 'function') { - cb = options - options = {} - } + before(async () => { + bigFile = Buffer.concat(await all(randomBytes(ONE_MEG * 1.2))) + smallFile = Buffer.concat(await all(randomBytes(200))) + }) + async function dagPut (options = {}) { options.type = options.type || 'file' options.content = options.content || Buffer.from([0x01, 0x02, 0x03]) options.links = options.links || [] const file = new UnixFS(options.type, options.content) - DAGNode.create(file.marshal(), options.links, (err, node) => { - expect(err).to.not.exist() - - ipld.put(node, { - version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb' - }, (err, cid) => { - cb(err, { file: file, node: node, cid: cid }) - }) + const node = await DAGNode.create(file.marshal(), options.links) + const cid = await ipld.put(node, mc.DAG_PB, { + cidVersion: 0, + hashAlg: mh.names['sha2-256'] }) - } - function addTestFile ({ file, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves }, cb) { - pull( - pull.values([{ - path, - content: file - }]), - importer(ipld, { - strategy, - rawLeaves, - chunkerOptions: { - maxChunkSize - } - }), - pull.collect((error, nodes) => { - cb(error, nodes && nodes[0] && nodes[0].multihash) - }) - ) + return { file: file, node: node, cid: cid } } - function addAndReadTestFile ({ file, offset, length, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves }, cb) { - addTestFile({ file, strategy, path, maxChunkSize, rawLeaves }, (error, multihash) => { - if (error) { - return cb(error) + async function addTestFile ({ file, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves }) { + const result = await all(importer([{ + path, + content: file + }], ipld, { + strategy, + rawLeaves, + chunkerOptions: { + maxChunkSize } + })) - pull( - exporter(multihash, ipld, { - offset, length - }), - pull.collect((error, files) => { - if (error) { - return cb(error) - } - - readFile(files[0], cb) - }) - ) - }) + return result[0].cid } - function addTestDirectory ({ directory, strategy = 'balanced', maxChunkSize }, callback) { - const input = push() - const dirName = path.basename(directory) - - pull( - input, - pull.map((file) => { - return { - path: path.join(dirName, path.basename(file)), - content: toPull.source(fs.createReadStream(file)) - } - }), - importer(ipld, { - strategy, - maxChunkSize - }), - pull.collect(callback) - ) + async function addAndReadTestFile ({ file, offset, length, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves }) { + const cid = await addTestFile({ file, strategy, path, maxChunkSize, rawLeaves }) + const entry = await exporter(cid, ipld) - const listFiles = (directory, depth, stream, cb) => { - waterfall([ - (done) => fs.stat(directory, done), - (stats, done) => { - if (stats.isDirectory()) { - return waterfall([ - (done) => fs.readdir(directory, done), - (children, done) => { - series( - children.map(child => (next) => listFiles(path.join(directory, child), depth + 1, stream, next)), - done - ) - } - ], done) - } - - stream.push(directory) - done() - } - ], cb) - } - - listFiles(directory, 0, input, () => { - input.end() - }) + return Buffer.concat(await all(entry.content({ + offset, length + }))) } - function checkBytesThatSpanBlocks (strategy, cb) { + async function checkBytesThatSpanBlocks (strategy) { const bytesInABlock = 262144 const bytes = Buffer.alloc(bytesInABlock + 100, 0) @@ -155,20 +81,14 @@ describe('exporter', () => { bytes[bytesInABlock] = 2 bytes[bytesInABlock + 1] = 3 - addAndReadTestFile({ + const data = await addAndReadTestFile({ file: bytes, offset: bytesInABlock - 1, length: 3, strategy - }, (error, data) => { - if (error) { - return cb(error) - } - - expect(data).to.deep.equal(Buffer.from([1, 2, 3])) - - cb() }) + + expect(data).to.deep.equal(Buffer.from([1, 2, 3])) } before((done) => { @@ -181,784 +101,523 @@ describe('exporter', () => { }) }) - it('ensure hash inputs are sanitized', (done) => { - dagPut((err, result) => { - expect(err).to.not.exist() - - ipld.get(result.cid, (err, res) => { - expect(err).to.not.exist() - const unmarsh = UnixFS.unmarshal(result.node.data) + it('ensure hash inputs are sanitized', async () => { + const result = await dagPut() + const node = await ipld.get(result.cid) + const unmarsh = UnixFS.unmarshal(node.data) - expect(unmarsh.data).to.deep.equal(result.file.data) + expect(unmarsh.data).to.deep.equal(result.file.data) - pull( - exporter(result.cid, ipld), - pull.collect(onFiles) - ) + const file = await exporter(result.cid, ipld) - function onFiles (err, files) { - expect(err).to.equal(null) - expect(files).to.have.length(1) - expect(files[0]).to.have.property('cid') - expect(files[0]).to.have.property('path', result.cid.toBaseEncodedString()) - fileEql(files[0], unmarsh.data, done) - } - }) - }) - }) + expect(file).to.have.property('cid') + expect(file).to.have.property('path', result.cid.toBaseEncodedString()) - it('exports a file with no links', (done) => { - dagPut((err, result) => { - expect(err).to.not.exist() - - pull( - zip( - pull( - ipld.getStream(result.cid), - pull.map((res) => UnixFS.unmarshal(res.value.data)) - ), - exporter(result.cid, ipld) - ), - pull.collect((err, values) => { - expect(err).to.not.exist() - const unmarsh = values[0][0] - const file = values[0][1] - - fileEql(file, unmarsh.data, done) - }) - ) - }) + const data = Buffer.concat(await all(file.content())) + expect(data).to.deep.equal(unmarsh.data) }) - it('small file in a directory with an escaped slash in the title', (done) => { + it('small file in a directory with an escaped slash in the title', async () => { const fileName = `small-\\/file-${Math.random()}.txt` const filePath = `/foo/${fileName}` - pull( - pull.values([{ - path: filePath, - content: pull.values([smallFile]) - }]), - importer(ipld), - pull.collect((err, files) => { - expect(err).to.not.exist() - - const path = `/ipfs/${new CID(files[1].multihash).toBaseEncodedString()}/${fileName}` - - pull( - exporter(path, ipld), - pull.collect((err, files) => { - expect(err).to.not.exist() - expect(files.length).to.equal(1) - expect(files[0].path).to.equal(fileName) - done() - }) - ) - }) - ) + const files = await all(importer([{ + path: filePath, + content: smallFile + }], ipld)) + + const path = `/ipfs/${files[1].cid.toBaseEncodedString()}/${fileName}` + const file = await exporter(path, ipld) + + expect(file.name).to.equal(fileName) + expect(file.path).to.equal(`${files[1].cid.toBaseEncodedString()}/${fileName}`) }) - it('small file in a directory with an square brackets in the title', (done) => { + it('small file in a directory with an square brackets in the title', async () => { const fileName = `small-[bar]-file-${Math.random()}.txt` const filePath = `/foo/${fileName}` - pull( - pull.values([{ - path: filePath, - content: pull.values([smallFile]) - }]), - importer(ipld), - pull.collect((err, files) => { - expect(err).to.not.exist() - - const path = `/ipfs/${new CID(files[1].multihash).toBaseEncodedString()}/${fileName}` - - pull( - exporter(path, ipld), - pull.collect((err, files) => { - expect(err).to.not.exist() - expect(files.length).to.equal(1) - expect(files[0].path).to.equal(fileName) - done() - }) - ) - }) - ) + const files = await all(importer([{ + path: filePath, + content: smallFile + }], ipld)) + + const path = `/ipfs/${files[1].cid.toBaseEncodedString()}/${fileName}` + const file = await exporter(path, ipld) + + expect(file.name).to.equal(fileName) + expect(file.path).to.equal(`${files[1].cid.toBaseEncodedString()}/${fileName}`) }) - it('exports a chunk of a file with no links', (done) => { + it('exports a chunk of a file with no links', async () => { const offset = 0 const length = 5 - dagPut({ - content: randomBytes(100) - }, (err, result) => { - expect(err).to.not.exist() - - pull( - zip( - pull( - ipld.getStream(result.cid), - pull.map((res) => UnixFS.unmarshal(res.value.data)) - ), - exporter(result.cid, ipld, { - offset, - length - }) - ), - pull.collect((err, values) => { - expect(err).to.not.exist() - - const unmarsh = values[0][0] - const file = values[0][1] - - fileEql(file, unmarsh.data.slice(offset, offset + length), done) - }) - ) + const result = await dagPut({ + content: Buffer.concat(await all(randomBytes(100))) }) + + const node = await ipld.get(result.cid) + const unmarsh = UnixFS.unmarshal(node.data) + + const file = await exporter(result.cid, ipld) + const data = Buffer.concat(await all(file.content({ + offset, + length + }))) + + expect(data).to.deep.equal(unmarsh.data.slice(offset, offset + length)) }) - it('exports a small file with links', function (done) { - waterfall([ - (cb) => dagPut({ content: randomBytes(100) }, cb), - (file, cb) => dagPut({ - content: randomBytes(100), - links: [ - new DAGLink('', file.node.size, file.cid) - ] - }, cb), - (result, cb) => { - pull( - exporter(result.cid, ipld), - pull.collect((err, files) => { - expect(err).to.not.exist() - - fileEql(files[0], result.file.data, cb) - }) - ) - } - ], done) + it('exports a small file with links', async () => { + const chunk = await dagPut({ content: randomBytes(100) }) + const result = await dagPut({ + content: Buffer.concat(await all(randomBytes(100))), + links: [ + await DAGLink.create('', chunk.node.size, chunk.cid) + ] + }) + + const file = await exporter(result.cid, ipld) + const data = Buffer.concat(await all(file.content())) + expect(data).to.deep.equal(result.file.data) }) - it('exports a chunk of a small file with links', function (done) { + it('exports a chunk of a small file with links', async () => { const offset = 0 const length = 5 - waterfall([ - (cb) => dagPut({ content: randomBytes(100) }, cb), - (file, cb) => dagPut({ - content: randomBytes(100), - links: [ - new DAGLink('', file.node.size, file.cid) - ] - }, cb), - (result, cb) => { - pull( - exporter(result.cid, ipld, { - offset, - length - }), - pull.collect((err, files) => { - expect(err).to.not.exist() - - fileEql(files[0], result.file.data.slice(offset, offset + length), cb) - }) - ) - } - ], done) + const chunk = await dagPut({ content: randomBytes(100) }) + const result = await dagPut({ + content: Buffer.concat(await all(randomBytes(100))), + links: [ + await DAGLink.create('', chunk.node.size, chunk.cid) + ] + }) + + const file = await exporter(result.cid, ipld) + const data = Buffer.concat(await all(file.content({ + offset, + length + }))) + + expect(data).to.deep.equal(result.file.data.slice(offset, offset + length)) }) - it('exports a large file > 5mb', function (done) { + it('exports a large file > 5mb', async function () { this.timeout(30 * 1000) - waterfall([ - (cb) => addTestFile({ - file: randomBytes(ONE_MEG * 6) - }, cb), - (buf, cb) => cb(null, new CID(buf)), - (cid, cb) => pull( - exporter(cid, ipld), - pull.collect((err, files) => cb(err, { cid, files })) - ), - ({ cid, files: [ file ] }, cb) => { - expect(file).to.have.property('path', cid.toBaseEncodedString()) - expect(file).to.have.property('size', ONE_MEG * 6) - fileEql(file, null, cb) - } - ], done) + const cid = await addTestFile({ + file: randomBytes(ONE_MEG * 6) + }) + + const file = await exporter(cid, ipld) + + expect(file).to.have.property('path', cid.toBaseEncodedString()) + expect(file.unixfs.fileSize()).to.equal(ONE_MEG * 6) }) - it('exports a chunk of a large file > 5mb', function (done) { + it('exports a chunk of a large file > 5mb', async function () { this.timeout(30 * 1000) const offset = 0 const length = 5 - const bytes = randomBytes(ONE_MEG * 6) - - waterfall([ - (cb) => addTestFile({ - file: bytes - }, cb), - (buf, cb) => cb(null, new CID(buf)), - (cid, cb) => pull( - exporter(cid, ipld, { - offset, - length - }), - pull.collect((err, files) => cb(err, { cid, files })) - ), - ({ cid, files: [ file ] }, cb) => { - expect(file).to.have.property('path', cid.toBaseEncodedString()) - - pull( - file.content, - pull.collect(cb) - ) - }, - ([ buf ], cb) => { - expect(buf).to.deep.equal(bytes.slice(offset, offset + length)) - cb() - } - ], done) + const bytes = Buffer.concat(await all(randomBytes(ONE_MEG * 6))) + + const cid = await addTestFile({ + file: bytes + }) + + const file = await exporter(cid, ipld) + expect(file).to.have.property('path', cid.toBaseEncodedString()) + + const data = Buffer.concat(await all(file.content({ + offset, + length + }))) + + expect(data).to.deep.equal(bytes.slice(offset, offset + length)) }) - it('exports the right chunks of files when offsets are specified', function (done) { + it('exports the right chunks of files when offsets are specified', async function () { this.timeout(30 * 1000) const offset = 3 const data = Buffer.alloc(300 * 1024) - addAndReadTestFile({ + const fileWithNoOffset = await addAndReadTestFile({ file: data, offset: 0 - }, (err, fileWithNoOffset) => { - expect(err).to.not.exist() - - addAndReadTestFile({ - file: data, - offset - }, (err, fileWithOffset) => { - expect(err).to.not.exist() - - expect(fileWithNoOffset.length).to.equal(data.length) - expect(fileWithNoOffset.length - fileWithOffset.length).to.equal(offset) - expect(fileWithOffset.length).to.equal(data.length - offset) - expect(fileWithNoOffset.length).to.equal(fileWithOffset.length + offset) + }) - done() - }) + const fileWithOffset = await addAndReadTestFile({ + file: data, + offset }) + + expect(fileWithNoOffset.length).to.equal(data.length) + expect(fileWithNoOffset.length - fileWithOffset.length).to.equal(offset) + expect(fileWithOffset.length).to.equal(data.length - offset) + expect(fileWithNoOffset.length).to.equal(fileWithOffset.length + offset) }) - it('exports a zero length chunk of a large file', function (done) { + it('exports a zero length chunk of a large file', async function () { this.timeout(30 * 1000) - addAndReadTestFile({ + const data = await addAndReadTestFile({ file: bigFile, path: '1.2MiB.txt', rawLeaves: true, length: 0 - }, (err, data) => { - expect(err).to.not.exist() - expect(data).to.eql(Buffer.alloc(0)) - done() }) - }) - - it('exports a directory', function (done) { - waterfall([ - (cb) => pull( - pull.values([{ - path: './200Bytes.txt', - content: randomBytes(ONE_MEG) - }, { - path: './dir-another' - }, { - path: './level-1/200Bytes.txt', - content: randomBytes(ONE_MEG) - }, { - path: './level-1/level-2' - }]), - importer(ipld), - pull.collect(cb) - ), - (files, cb) => cb(null, files.pop().multihash), - (buf, cb) => cb(null, new CID(buf)), - (cid, cb) => pull( - exporter(cid, ipld), - pull.collect((err, files) => cb(err, { cid, files })) - ), - ({ cid, files }, cb) => { - files.forEach(file => expect(file).to.have.property('cid')) - - expect( - files.map((file) => file.path) - ).to.be.eql([ - cid.toBaseEncodedString(), - `${cid.toBaseEncodedString()}/200Bytes.txt`, - `${cid.toBaseEncodedString()}/dir-another`, - `${cid.toBaseEncodedString()}/level-1`, - `${cid.toBaseEncodedString()}/level-1/200Bytes.txt`, - `${cid.toBaseEncodedString()}/level-1/level-2` - ]) - - files - .filter(file => file.type === 'dir') - .forEach(dir => { - expect(dir).to.has.property('size', 0) - }) - - pull( - pull.values(files), - pull.map((file) => Boolean(file.content)), - pull.collect(cb) - ) - }, - (contents, cb) => { - expect(contents).to.be.eql([ - false, - true, - false, - false, - true, - false - ]) - cb() - } - ], done) - }) - it('exports a directory one deep', function (done) { - waterfall([ - (cb) => pull( - pull.values([{ - path: './200Bytes.txt', - content: randomBytes(ONE_MEG) - }, { - path: './dir-another' - }, { - path: './level-1' - }]), - importer(ipld), - pull.collect(cb) - ), - (files, cb) => cb(null, files.pop().multihash), - (buf, cb) => cb(null, new CID(buf)), - (cid, cb) => pull( - exporter(cid, ipld), - pull.collect((err, files) => cb(err, { cid, files })) - ), - ({ cid, files }, cb) => { - files.forEach(file => expect(file).to.have.property('cid')) - - expect( - files.map((file) => file.path) - ).to.be.eql([ - cid.toBaseEncodedString(), - `${cid.toBaseEncodedString()}/200Bytes.txt`, - `${cid.toBaseEncodedString()}/dir-another`, - `${cid.toBaseEncodedString()}/level-1` - ]) - - pull( - pull.values(files), - pull.map((file) => Boolean(file.content)), - pull.collect(cb) - ) - }, - (contents, cb) => { - expect(contents).to.be.eql([ - false, - true, - false, - false - ]) - cb() - } - ], done) - }) + expect(data).to.eql(Buffer.alloc(0)) + }) + + it('exports a directory', async () => { + const importedDir = await last(importer([{ + path: './200Bytes.txt', + content: randomBytes(ONE_MEG) + }, { + path: './dir-another' + }, { + path: './level-1/200Bytes.txt', + content: randomBytes(ONE_MEG) + }, { + path: './level-1/level-2' + }], ipld)) + const dir = await exporter(importedDir.cid, ipld) + const files = await all(dir.content()) + + files.forEach(file => expect(file).to.have.property('cid')) + + expect( + files.map((file) => file.path) + ).to.be.eql([ + `${dir.cid.toBaseEncodedString()}/200Bytes.txt`, + `${dir.cid.toBaseEncodedString()}/dir-another`, + `${dir.cid.toBaseEncodedString()}/level-1` + ]) + + files + .filter(file => file.unixfs.type === 'dir') + .forEach(dir => { + expect(dir).to.has.property('size', 0) + }) - it('exports a small file imported with raw leaves', function (done) { + expect( + files + .map(file => file.unixfs.type === 'file') + ).to.deep.equal([ + true, + false, + false + ]) + }) + + it('exports a directory one deep', async () => { + const importedDir = await last(importer([{ + path: './200Bytes.txt', + content: randomBytes(ONE_MEG) + }, { + path: './dir-another' + }, { + path: './level-1' + }], ipld)) + + const dir = await exporter(importedDir.cid, ipld) + const files = await all(dir.content()) + + files.forEach(file => expect(file).to.have.property('cid')) + + expect( + files.map((file) => file.path) + ).to.be.eql([ + `${importedDir.cid.toBaseEncodedString()}/200Bytes.txt`, + `${importedDir.cid.toBaseEncodedString()}/dir-another`, + `${importedDir.cid.toBaseEncodedString()}/level-1` + ]) + + expect( + files + .map(file => file.unixfs.type === 'file') + ).to.deep.equal([ + true, + false, + false + ]) + }) + + it('exports a small file imported with raw leaves', async function () { this.timeout(30 * 1000) - addAndReadTestFile({ + const data = await addAndReadTestFile({ file: smallFile, path: '200Bytes.txt', rawLeaves: true - }, (err, data) => { - expect(err).to.not.exist() - expect(data).to.eql(smallFile) - done() }) + + expect(data).to.deep.equal(smallFile) }) - it('exports a chunk of a small file imported with raw leaves', function (done) { + it('exports a chunk of a small file imported with raw leaves', async function () { this.timeout(30 * 1000) const length = 100 - addAndReadTestFile({ + const data = await addAndReadTestFile({ file: smallFile, path: '200Bytes.txt', rawLeaves: true, length - }, (err, data) => { - expect(err).to.not.exist() - expect(data).to.eql(smallFile.slice(0, length)) - done() }) + + expect(data).to.eql(smallFile.slice(0, length)) }) - it('exports a chunk of a small file imported with raw leaves with length', function (done) { + it('exports a chunk of a small file imported with raw leaves with length', async function () { this.timeout(30 * 1000) const offset = 100 const length = 200 - addAndReadTestFile({ + const data = await addAndReadTestFile({ file: smallFile, path: '200Bytes.txt', rawLeaves: true, offset, length - }, (err, data) => { - expect(err).to.not.exist() - expect(data).to.eql(smallFile.slice(offset)) - done() }) + + expect(data).to.eql(smallFile.slice(offset)) }) - it('exports a zero length chunk of a small file imported with raw leaves', function (done) { + it('exports a zero length chunk of a small file imported with raw leaves', async function () { this.timeout(30 * 1000) const length = 0 - addAndReadTestFile({ + const data = await addAndReadTestFile({ file: smallFile, path: '200Bytes.txt', rawLeaves: true, length - }, (err, data) => { - expect(err).to.not.exist() - expect(data).to.eql(Buffer.alloc(0)) - done() }) + + expect(data).to.eql(Buffer.alloc(0)) }) - it('errors when exporting a chunk of a small file imported with raw leaves and negative length', function (done) { + it('errors when exporting a chunk of a small file imported with raw leaves and negative length', async function () { this.timeout(30 * 1000) const length = -100 - addAndReadTestFile({ - file: smallFile, - path: '200Bytes.txt', - rawLeaves: true, - length - }, (err, data) => { - expect(err).to.exist() + try { + await addAndReadTestFile({ + file: smallFile, + path: '200Bytes.txt', + rawLeaves: true, + length + }) + throw new Error('Should not have got this far') + } catch (err) { expect(err.message).to.equal('Length must be greater than or equal to 0') - done() - }) + expect(err.code).to.equal('EINVALIDPARAMS') + } }) - it('errors when exporting a chunk of a small file imported with raw leaves and negative offset', function (done) { + it('errors when exporting a chunk of a small file imported with raw leaves and negative offset', async function () { this.timeout(30 * 1000) const offset = -100 - addAndReadTestFile({ - file: smallFile, - path: '200Bytes.txt', - rawLeaves: true, - offset - }, (err, data) => { - expect(err).to.exist() + try { + await addAndReadTestFile({ + file: smallFile, + path: '200Bytes.txt', + rawLeaves: true, + offset + }) + throw new Error('Should not have got this far') + } catch (err) { expect(err.message).to.equal('Offset must be greater than or equal to 0') - done() - }) + expect(err.code).to.equal('EINVALIDPARAMS') + } }) - it('errors when exporting a chunk of a small file imported with raw leaves and offset greater than file size', function (done) { + it('errors when exporting a chunk of a small file imported with raw leaves and offset greater than file size', async function () { this.timeout(30 * 1000) const offset = 201 - addAndReadTestFile({ - file: smallFile, - path: '200Bytes.txt', - rawLeaves: true, - offset - }, (err, data) => { - expect(err).to.exist() + try { + await addAndReadTestFile({ + file: smallFile, + path: '200Bytes.txt', + rawLeaves: true, + offset + }) + throw new Error('Should not have got this far') + } catch (err) { expect(err.message).to.equal('Offset must be less than the file size') - done() - }) + expect(err.code).to.equal('EINVALIDPARAMS') + } }) - it('exports a large file > 1mb imported with raw leaves', function (done) { - waterfall([ - (cb) => pull( - pull.values([{ - path: '1.2MiB.txt', - content: pull.values([bigFile]) - }]), - importer(ipld, { - rawLeaves: true - }), - pull.collect(cb) - ), - (files, cb) => { - expect(files.length).to.equal(1) - - pull( - exporter(files[0].multihash, ipld), - pull.collect(cb) - ) - }, - (files, cb) => { - fileEql(files[0], bigFile, done) - } - ], done) - }) + it('exports a large file > 1mb imported with raw leaves', async () => { + const imported = await first(importer([{ + path: '1.2MiB.txt', + content: bigFile + }], ipld, { + rawLeaves: true + })) - it('returns an empty stream for dir', (done) => { - const hash = 'QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' + const file = await exporter(imported.cid, ipld) + const data = Buffer.concat(await all(file.content())) - pull( - exporter(hash, ipld), - pull.collect((err, files) => { - expect(err).to.not.exist() - expect(files[0].content).to.not.exist() - done() - }) - ) + expect(data).to.deep.equal(bigFile) }) - it('reads bytes with an offset', (done) => { - addAndReadTestFile({ + it('returns an empty stream for dir', async () => { + const imported = await first(importer([{ + path: 'empty' + }], ipld)) + const dir = await exporter(imported.cid, ipld) + const files = await all(dir.content()) + expect(files.length).to.equal(0) + }) + + it('reads bytes with an offset', async () => { + const data = await addAndReadTestFile({ file: Buffer.from([0, 1, 2, 3]), offset: 1 - }, (error, data) => { - expect(error).to.not.exist() - expect(data).to.deep.equal(Buffer.from([1, 2, 3])) - - done() }) - }) - it('reads bytes with a negative offset', (done) => { - addAndReadTestFile({ - file: Buffer.from([0, 1, 2, 3]), - offset: -1 - }, (error, data) => { - expect(error).to.be.ok() - expect(error.message).to.contain('Offset must be greater than or equal to 0') + expect(data).to.deep.equal(Buffer.from([1, 2, 3])) + }) - done() - }) + it('errors when reading bytes with a negative offset', async () => { + try { + await addAndReadTestFile({ + file: Buffer.from([0, 1, 2, 3]), + offset: -1 + }) + throw new Error('Should not have got this far') + } catch (err) { + expect(err.message).to.contain('Offset must be greater than or equal to 0') + expect(err.code).to.equal('EINVALIDPARAMS') + } }) - it('reads bytes with an offset and a length', (done) => { - addAndReadTestFile({ + it('reads bytes with an offset and a length', async () => { + const data = await addAndReadTestFile({ file: Buffer.from([0, 1, 2, 3]), offset: 0, length: 1 - }, (error, data) => { - expect(error).to.not.exist() - expect(data).to.deep.equal(Buffer.from([0])) - - done() }) - }) - it('reads bytes with a negative length', (done) => { - addAndReadTestFile({ - file: Buffer.from([0, 1, 2, 3, 4]), - offset: 2, - length: -1 - }, (error, data) => { - expect(error).to.be.ok() - expect(error.message).to.contain('Length must be greater than or equal to 0') + expect(data).to.deep.equal(Buffer.from([0])) + }) - done() - }) + it('errors when reading bytes with a negative length', async () => { + try { + await addAndReadTestFile({ + file: Buffer.from([0, 1, 2, 3, 4]), + offset: 2, + length: -1 + }) + } catch (err) { + expect(err.message).to.contain('Length must be greater than or equal to 0') + expect(err.code).to.equal('EINVALIDPARAMS') + } }) - it('reads bytes with an offset and a length', (done) => { - addAndReadTestFile({ + it('reads bytes with an offset and a length', async () => { + const data = await addAndReadTestFile({ file: Buffer.from([0, 1, 2, 3, 4]), offset: 1, length: 4 - }, (error, data) => { - expect(error).to.not.exist() - expect(data).to.deep.equal(Buffer.from([1, 2, 3, 4])) - - done() }) + + expect(data).to.deep.equal(Buffer.from([1, 2, 3, 4])) }) - it('reads files that are split across lots of nodes', function (done) { + it('reads files that are split across lots of nodes', async function () { this.timeout(30 * 1000) - addAndReadTestFile({ + const data = await addAndReadTestFile({ file: bigFile, offset: 0, length: bigFile.length, maxChunkSize: 1024 - }, (error, data) => { - expect(error).to.not.exist() - expect(data).to.deep.equal(bigFile) - - done() }) + + expect(data).to.deep.equal(bigFile) }) - it('reads files in multiple steps that are split across lots of nodes in really small chunks', function (done) { + it('reads files in multiple steps that are split across lots of nodes in really small chunks', async function () { this.timeout(600 * 1000) let results = [] let chunkSize = 1024 let offset = 0 - addTestFile({ + const cid = await addTestFile({ file: bigFile, maxChunkSize: 1024 - }, (error, multihash) => { - expect(error).to.not.exist() - - doUntil( - (cb) => { - waterfall([ - (next) => { - pull( - exporter(multihash, ipld, { - offset, - length: chunkSize - }), - pull.collect(next) - ) - }, - (files, next) => readFile(files[0], next) - ], cb) - }, - (result) => { - results.push(result) - - offset += result.length - - return offset >= bigFile.length - }, - (error) => { - expect(error).to.not.exist() - - const buffer = Buffer.concat(results) - - expect(buffer).to.deep.equal(bigFile) - - done() - } - ) }) - }) + const file = await exporter(cid, ipld) - it('reads bytes with an offset and a length that span blocks using balanced layout', (done) => { - checkBytesThatSpanBlocks('balanced', done) - }) + while (offset < bigFile.length) { + const result = Buffer.concat(await all(file.content({ + offset, + length: chunkSize + }))) + results.push(result) - it('reads bytes with an offset and a length that span blocks using flat layout', (done) => { - checkBytesThatSpanBlocks('flat', done) + offset += result.length + } + + const buffer = Buffer.concat(results) + + expect(buffer).to.deep.equal(bigFile) }) - it('reads bytes with an offset and a length that span blocks using trickle layout', (done) => { - checkBytesThatSpanBlocks('trickle', done) + it('reads bytes with an offset and a length that span blocks using balanced layout', async () => { + await checkBytesThatSpanBlocks('balanced') }) - it('exports a directory containing an empty file whose content gets turned into a ReadableStream', function (done) { - if (!isNode) { - return this.skip() - } + it('reads bytes with an offset and a length that span blocks using flat layout', async () => { + await checkBytesThatSpanBlocks('flat') + }) - // replicates the behaviour of ipfs.files.get - waterfall([ - (cb) => addTestDirectory({ - directory: path.join(__dirname, 'fixtures', 'dir-with-empty-files') - }, cb), - (result, cb) => { - const dir = result.pop() - - pull( - exporter(dir.multihash, ipld), - pull.map((file) => { - if (file.content) { - file.content = toStream.source(file.content) - file.content.pause() - } - - return file - }), - pull.collect((error, files) => { - if (error) { - return cb(error) - } - - series( - files - .filter(file => Boolean(file.content)) - .map(file => { - return (done) => { - if (file.content) { - file.content - .pipe(toStream.sink(pull.collect((error, bufs) => { - expect(error).to.not.exist() - expect(bufs.length).to.equal(1) - expect(bufs[0].length).to.equal(0) - - done() - }))) - } - } - }), - cb - ) - }) - ) - } - ], done) + it('reads bytes with an offset and a length that span blocks using trickle layout', async () => { + await checkBytesThatSpanBlocks('trickle') }) - it('fails on non existent hash', (done) => { + it('fails on non existent hash', async () => { // This hash doesn't exist in the repo const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKj3' - pull( - exporter(hash, ipld), - pull.collect((err, files) => { - expect(err).to.exist() - done() - }) - ) + try { + await exporter(hash, ipld) + } catch (err) { + expect(err.code).to.equal('ERR_NOT_FOUND') + } }) - it('exports file with data on internal and leaf nodes', function (done) { - waterfall([ - (cb) => createAndPersistNode(ipld, 'raw', [0x04, 0x05, 0x06, 0x07], [], cb), - (leaf, cb) => createAndPersistNode(ipld, 'file', [0x00, 0x01, 0x02, 0x03], [ - leaf - ], cb), - (file, cb) => { - pull( - exporter(file.cid, ipld), - pull.asyncMap((file, cb) => readFile(file, cb)), - pull.through(buffer => { - expect(buffer).to.deep.equal(Buffer.from([0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07])) - }), - pull.collect(cb) - ) - } - ], done) + it('exports file with data on internal and leaf nodes', async () => { + const leaf = await createAndPersistNode(ipld, 'raw', [0x04, 0x05, 0x06, 0x07], []) + const node = await createAndPersistNode(ipld, 'file', [0x00, 0x01, 0x02, 0x03], [ + leaf + ]) + + const file = await exporter(node.cid, ipld) + const data = Buffer.concat(await all(file.content())) + + expect(data).to.deep.equal(Buffer.from([0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07])) }) - it('exports file with data on some internal and leaf nodes', function (done) { + it('exports file with data on some internal and leaf nodes', async () => { // create a file node with three children: // where: // i = internal node without data @@ -969,197 +628,103 @@ describe('exporter', () => { // l d i // | \ // l l - waterfall([ - (cb) => { - // create leaves - parallel([ - (next) => createAndPersistNode(ipld, 'raw', [0x00, 0x01, 0x02, 0x03], [], next), - (next) => createAndPersistNode(ipld, 'raw', [0x08, 0x09, 0x10, 0x11], [], next), - (next) => createAndPersistNode(ipld, 'raw', [0x12, 0x13, 0x14, 0x15], [], next) - ], cb) - }, - (leaves, cb) => { - parallel([ - (next) => createAndPersistNode(ipld, 'raw', [0x04, 0x05, 0x06, 0x07], [leaves[1]], next), - (next) => createAndPersistNode(ipld, 'raw', null, [leaves[2]], next) - ], (error, internalNodes) => { - if (error) { - return cb(error) - } - - createAndPersistNode(ipld, 'file', null, [ - leaves[0], - internalNodes[0], - internalNodes[1] - ], cb) - }) - }, - (file, cb) => { - pull( - exporter(file.cid, ipld), - pull.asyncMap((file, cb) => readFile(file, cb)), - pull.through(buffer => { - expect(buffer).to.deep.equal( - Buffer.from([ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15 - ]) - ) - }), - pull.collect(cb) - ) - } - ], done) + const leaves = await Promise.all([ + createAndPersistNode(ipld, 'raw', [0x00, 0x01, 0x02, 0x03], []), + createAndPersistNode(ipld, 'raw', [0x08, 0x09, 0x10, 0x11], []), + createAndPersistNode(ipld, 'raw', [0x12, 0x13, 0x14, 0x15], []) + ]) + + const internalNodes = await Promise.all([ + createAndPersistNode(ipld, 'raw', [0x04, 0x05, 0x06, 0x07], [leaves[1]]), + createAndPersistNode(ipld, 'raw', null, [leaves[2]]) + ]) + + const node = await createAndPersistNode(ipld, 'file', null, [ + leaves[0], + internalNodes[0], + internalNodes[1] + ]) + + const file = await exporter(node.cid, ipld) + const data = Buffer.concat(await all(file.content())) + + expect(data).to.deep.equal( + Buffer.from([ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15 + ]) + ) }) - it('exports file with data on internal and leaf nodes with an offset that only fetches data from leaf nodes', function (done) { - waterfall([ - (cb) => createAndPersistNode(ipld, 'raw', [0x04, 0x05, 0x06, 0x07], [], cb), - (leaf, cb) => createAndPersistNode(ipld, 'file', [0x00, 0x01, 0x02, 0x03], [ - leaf - ], cb), - (file, cb) => { - pull( - exporter(file.cid, ipld, { - offset: 4 - }), - pull.asyncMap((file, cb) => readFile(file, cb)), - pull.through(buffer => { - expect(buffer).to.deep.equal(Buffer.from([0x04, 0x05, 0x06, 0x07])) - }), - pull.collect(cb) - ) - } - ], done) + it('exports file with data on internal and leaf nodes with an offset that only fetches data from leaf nodes', async () => { + const leaf = await createAndPersistNode(ipld, 'raw', [0x04, 0x05, 0x06, 0x07], []) + const node = await createAndPersistNode(ipld, 'file', [0x00, 0x01, 0x02, 0x03], [ + leaf + ]) + + const file = await exporter(node.cid, ipld) + const data = Buffer.concat(await all(file.content({ + offset: 4 + }))) + + expect(data).to.deep.equal(Buffer.from([0x04, 0x05, 0x06, 0x07])) }) - it('exports file with data on leaf nodes without emitting empty buffers', function (done) { + it('exports file with data on leaf nodes without emitting empty buffers', async function () { this.timeout(30 * 1000) - pull( - pull.values([{ - path: '200Bytes.txt', - content: pull.values([bigFile]) - }]), - importer(ipld, { - rawLeaves: true - }), - pull.collect(collected) - ) - - function collected (err, files) { - expect(err).to.not.exist() - expect(files.length).to.equal(1) - - pull( - exporter(files[0].multihash, ipld), - pull.collect((err, files) => { - expect(err).to.not.exist() - expect(files.length).to.equal(1) - - pull( - files[0].content, - pull.collect((error, buffers) => { - expect(error).to.not.exist() - - buffers.forEach(buffer => { - expect(buffer.length).to.not.equal(0) - }) - - done() - }) - ) - }) - ) - } - }) + const imported = await first(importer([{ + path: '200Bytes.txt', + content: bigFile + }], ipld, { + rawLeaves: true + })) - it('exports a raw leaf', (done) => { - pull( - pull.values([{ - path: '200Bytes.txt', - content: pull.values([smallFile]) - }]), - importer(ipld, { - rawLeaves: true - }), - pull.collect(collected) - ) + const file = await exporter(imported.cid, ipld) + const buffers = await all(file.content()) - function collected (err, files) { - expect(err).to.not.exist() - expect(files.length).to.equal(1) - - pull( - exporter(files[0].multihash, ipld), - pull.collect((err, files) => { - expect(err).to.not.exist() - expect(files.length).to.equal(1) - expect(CID.isCID(files[0].cid)).to.be.true() - fileEql(files[0], smallFile, done) - }) - ) - } + buffers.forEach(buffer => { + expect(buffer.length).to.not.equal(0) + }) }) -}) -function fileEql (actual, expected, done) { - readFile(actual, (error, data) => { - if (error) { - return done(error) - } + it('exports a raw leaf', async () => { + const imported = await first(importer([{ + path: '200Bytes.txt', + content: smallFile + }], ipld, { + rawLeaves: true + })) - try { - if (expected) { - expect(data).to.eql(expected) - } else { - expect(data).to.exist() - } - } catch (err) { - return done(err) - } + const file = await exporter(imported.cid, ipld) + expect(CID.isCID(file.cid)).to.be.true() - done() + const data = Buffer.concat(await all(file.content())) + expect(data).to.deep.equal(smallFile) }) -} - -function readFile (file, done) { - pull( - file.content, - pull.collect((error, data) => { - if (error) { - return done(error) - } - - done(null, Buffer.concat(data)) - }) - ) -} +}) -function createAndPersistNode (ipld, type, data, children, callback) { +async function createAndPersistNode (ipld, type, data, children) { const file = new UnixFS(type, data ? Buffer.from(data) : undefined) const links = [] - children.forEach(child => { + for (let i = 0; i < children.length; i++) { + const child = children[i] const leaf = UnixFS.unmarshal(child.node.data) file.addBlockSize(leaf.fileSize()) - links.push(new DAGLink('', child.node.size, child.cid)) - }) - - DAGNode.create(file.marshal(), links, (error, node) => { - if (error) { - return callback(error) - } + links.push(await DAGLink.create('', child.node.size, child.cid)) + } - ipld.put(node, { - version: 1, - hashAlg: 'sha2-256', - format: 'dag-pb' - }, (error, cid) => callback(error, { - node, - cid - })) + const node = await DAGNode.create(file.marshal(), links) + const cid = await ipld.put(node, mc.DAG_PB, { + cidVersion: 1, + hashAlg: mh.names['sha2-256'] }) + + return { + node, + cid + } } diff --git a/test/helpers/create-shard.js b/test/helpers/create-shard.js new file mode 100644 index 0000000..96cd9e9 --- /dev/null +++ b/test/helpers/create-shard.js @@ -0,0 +1,33 @@ +'use strict' + +const importer = require('ipfs-unixfs-importer') + +const SHARD_SPLIT_THRESHOLD = 10 + +const createShard = async (numFiles, ipld) => { + return createShardWithFileNames(numFiles, (index) => `file-${index}`, ipld) +} + +const createShardWithFileNames = async (numFiles, fileName, ipld) => { + const files = new Array(numFiles).fill(0).map((_, index) => ({ + path: fileName(index), + content: Buffer.from([0, 1, 2, 3, 4, index]) + })) + + return createShardWithFiles(files, ipld) +} + +const createShardWithFiles = async (files, ipld) => { + let last + + for await (const imported of importer(ipld, files, { + shardSplitThreshold: SHARD_SPLIT_THRESHOLD, + wrap: true + })) { + last = imported + } + + return last.cid +} + +module.exports = createShard diff --git a/test/helpers/dag-pb.js b/test/helpers/dag-pb.js new file mode 100644 index 0000000..947422b --- /dev/null +++ b/test/helpers/dag-pb.js @@ -0,0 +1,30 @@ +'use strict' + +const { + DAGNode, + DAGLink +} = require('ipld-dag-pb') +const promisify = require('promisify-es6') + +const node = { + create: promisify(DAGNode.create, { + context: DAGNode + }), + addLink: promisify(DAGNode.addLink, { + context: DAGNode + }), + rmLink: promisify(DAGNode.rmLink, { + context: DAGNode + }) +} + +const link = { + create: promisify(DAGLink.create, { + context: DAGLink + }) +} + +module.exports = { + DAGNode: node, + DAGLink: link +} diff --git a/test/helpers/random-bytes.js b/test/helpers/random-bytes.js deleted file mode 100644 index 1c10604..0000000 --- a/test/helpers/random-bytes.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict' - -const crypto = require('crypto') -const MAX_BYTES = 65536 - -// One day this will be merged: https://github.com/crypto-browserify/randombytes/pull/16 -module.exports = function randomBytes (num) { - num = parseInt(num) - const bytes = Buffer.allocUnsafe(num) - - for (let offset = 0; offset < num; offset += MAX_BYTES) { - let size = MAX_BYTES - - if ((offset + size) > num) { - size = num - offset - } - - crypto.randomFillSync(bytes, offset, size) - } - - return bytes -} From eea84236a119960ae0c97315b8329e4bd7c20d99 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 25 Apr 2019 16:45:23 +0100 Subject: [PATCH 02/11] chore: readme update --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index da2992d..73181c4 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,12 @@ - [Usage](#usage) - [Example](#example) - [API](#api) - - [exporter(cid, ipld)](#exportercid-ipld) + - [`exporter(cid, ipld)`](#exportercid-ipld) - [UnixFS V1 entries](#unixfs-v1-entries) - [Raw entries](#raw-entries) - [CBOR entries](#cbor-entries) - - [`entry.content({ offset, length })`](#entrycontent-offset-length) - - [exporter.path(cid, ipld)](#exporterpathcid-ipld) + - [`entry.content({ offset, length })`](#entrycontent-offset-length) + - [`exporter.path(cid, ipld)`](#exporterpathcid-ipld) - [Contribute](#contribute) - [License](#license) @@ -89,7 +89,7 @@ console.info(content) // 0, 1, 2, 3 const exporter = require('ipfs-unixfs-exporter') ``` -### exporter(cid, ipld) +### `exporter(cid, ipld)` Uses the given [js-ipld instance][] to fetch an IPFS node by it's CID. @@ -164,7 +164,7 @@ Entries with a `dag-cbor` codec `CID` return JavaScript object entries: There is no `content` function for a `CBOR` node. -##### `entry.content({ offset, length })` +#### `entry.content({ offset, length })` When `entry` is a file or a `raw` node, `offset` and/or `length` arguments can be passed to `entry.content()` to return slices of data: @@ -197,7 +197,7 @@ for await (const entry of dir.content({ // `entries` contains the first 5 files/directories in the directory ``` -### exporter.path(cid, ipld) +### `exporter.path(cid, ipld)` `exporter.path` will return an async iterator that yields entries for all segments in a path: From e1334bb3d66620afe87f84746ad145cea27dcc88 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 25 Apr 2019 16:53:12 +0100 Subject: [PATCH 03/11] chore: fix linting --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index c00c0ea..97258a5 100644 --- a/package.json +++ b/package.json @@ -47,8 +47,8 @@ "ipld": "~0.22.0", "ipld-dag-pb": "~0.15.2", "ipld-in-memory": "^2.0.0", - "multicodec": "^0.5.1", - "multihashes": "^0.4.14", + "multicodec": "~0.5.1", + "multihashes": "~0.4.14", "pull-pushable": "^2.2.0", "pull-stream-to-stream": "^1.3.4", "pull-zip": "^2.0.1", From 1c7456bd08bad09504de1ab76aed76df18a61e92 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 26 Apr 2019 09:32:02 +0100 Subject: [PATCH 04/11] feat: convert internals to be async/await --- package.json | 15 +- src/resolvers/raw.js | 40 +---- src/resolvers/unixfs-v1/content/file.js | 195 ++++++------------------ src/resolvers/unixfs-v1/content/raw.js | 38 +---- src/utils/validate-offset-and-length.js | 36 +++++ test/exporter.spec.js | 118 +++++++++----- 6 files changed, 184 insertions(+), 258 deletions(-) create mode 100644 src/utils/validate-offset-and-length.js diff --git a/package.json b/package.json index 97258a5..ecf43dd 100644 --- a/package.json +++ b/package.json @@ -49,27 +49,16 @@ "ipld-in-memory": "^2.0.0", "multicodec": "~0.5.1", "multihashes": "~0.4.14", - "pull-pushable": "^2.2.0", - "pull-stream-to-stream": "^1.3.4", - "pull-zip": "^2.0.1", - "sinon": "^7.1.0", - "stream-to-pull-stream": "^1.7.2" + "sinon": "^7.1.0" }, "dependencies": { - "async": "^2.6.1", "async-iterator-last": "0.0.2", "cids": "~0.5.5", "err-code": "^1.1.2", "hamt-sharding": "0.0.2", "ipfs-unixfs": "~0.1.16", "ipfs-unixfs-importer": "ipfs/js-ipfs-unixfs-importer#async-await", - "promisify-es6": "^1.0.3", - "pull-cat": "^1.1.11", - "pull-defer": "~0.2.3", - "pull-paramap": "^1.2.2", - "pull-stream": "^3.6.9", - "pull-stream-to-async-iterator": "^1.0.1", - "pull-traverse": "^1.0.3" + "promisify-es6": "^1.0.3" }, "contributors": [ "Alan Shaw ", diff --git a/src/resolvers/raw.js b/src/resolvers/raw.js index ca85677..ae7d821 100644 --- a/src/resolvers/raw.js +++ b/src/resolvers/raw.js @@ -2,42 +2,16 @@ const errCode = require('err-code') const extractDataFromBlock = require('../utils/extract-data-from-block') -const toIterator = require('pull-stream-to-async-iterator') -const once = require('pull-stream/sources/once') -const error = require('pull-stream/sources/error') +const validateOffsetAndLength = require('../utils/validate-offset-and-length') const rawContent = (node) => { - return (options = {}) => { - const size = node.length + return async function * (options = {}) { + const { + offset, + length + } = validateOffsetAndLength(node.length, options.offset, options.length) - let offset = options.offset - let length = options.length - - if (offset < 0) { - return toIterator(error(errCode(new Error('Offset must be greater than or equal to 0'), 'EINVALIDPARAMS'))) - } - - if (offset > size) { - return toIterator(error(errCode(new Error('Offset must be less than the file size'), 'EINVALIDPARAMS'))) - } - - if (length < 0) { - return toIterator(error(errCode(new Error('Length must be greater than or equal to 0'), 'EINVALIDPARAMS'))) - } - - if (length === 0) { - return toIterator(once(Buffer.alloc(0))) - } - - if (!offset) { - offset = 0 - } - - if (!length || (offset + length > size)) { - length = size - offset - } - - return toIterator(once(extractDataFromBlock(node, 0, offset, offset + length))) + yield extractDataFromBlock(node, 0, offset, offset + length) } } diff --git a/src/resolvers/unixfs-v1/content/file.js b/src/resolvers/unixfs-v1/content/file.js index 1662204..421ec23 100644 --- a/src/resolvers/unixfs-v1/content/file.js +++ b/src/resolvers/unixfs-v1/content/file.js @@ -1,180 +1,81 @@ 'use strict' const extractDataFromBlock = require('../../../utils/extract-data-from-block') -const toIterator = require('pull-stream-to-async-iterator') -const traverse = require('pull-traverse') +const validateOffsetAndLength = require('../../../utils/validate-offset-and-length') const UnixFS = require('ipfs-unixfs') -const pull = require('pull-stream/pull') -const error = require('pull-stream/sources/error') -const once = require('pull-stream/sources/once') -const empty = require('pull-stream/sources/empty') -const filter = require('pull-stream/throughs/filter') -const flatten = require('pull-stream/throughs/flatten') -const map = require('pull-stream/throughs/map') -const paramap = require('pull-paramap') -const errCode = require('err-code') - -function streamBytes (ipld, node, fileSize, { offset, length }) { - if (offset === fileSize || length === 0) { - return once(Buffer.alloc(0)) - } - if (!offset) { - offset = 0 - } +async function * emitBytes (ipld, node, start, end, streamPosition = 0) { + // a `raw` node + if (Buffer.isBuffer(node)) { + const buf = extractDataFromBlock(node, streamPosition, start, end) - if (!length) { - length = fileSize - } + if (buf.length) { + yield buf + } - if (offset < 0) { - return error(errCode(new Error('Offset must be greater than or equal to 0'), 'EINVALIDPARAMS')) - } + streamPosition += buf.length - if (offset > fileSize) { - return error(errCode(new Error('Offset must be less than the file size'), 'EINVALIDPARAMS')) + return streamPosition } - if (length < 0) { - return error(errCode(new Error('Length must be greater than or equal to 0'), 'EINVALIDPARAMS')) - } + let file - const end = offset + length - - return pull( - traverse.depthFirst({ - node, - start: 0, - end: fileSize - }, getChildren(ipld, offset, end)), - map(extractData(offset, end)), - filter(Boolean) - ) -} - -function getChildren (dag, offset, end) { - // as we step through the children, keep track of where we are in the stream - // so we can filter out nodes we're not interested in - let streamPosition = 0 - - return function visitor ({ node }) { - if (Buffer.isBuffer(node)) { - // this is a leaf node, can't traverse any further - return empty() - } - - let file - - try { - file = UnixFS.unmarshal(node.data) - } catch (err) { - return error(err) - } + try { + file = UnixFS.unmarshal(node.data) + } catch (err) { + throw err + } - const nodeHasData = Boolean(file.data && file.data.length) + // might be a unixfs `raw` node or have data on intermediate nodes + const nodeHasData = Boolean(file.data && file.data.length) - // handle case where data is present on leaf nodes and internal nodes - if (nodeHasData && node.links.length) { - streamPosition += file.data.length - } + if (nodeHasData) { + const buf = extractDataFromBlock(file.data, streamPosition, start, end) - // work out which child nodes contain the requested data - const filteredLinks = node.links - .map((link, index) => { - const child = { - link: link, - start: streamPosition, - end: streamPosition + file.blockSizes[index], - size: file.blockSizes[index] - } - - streamPosition = child.end - - return child - }) - .filter((child) => { - return (offset >= child.start && offset < child.end) || // child has offset byte - (end > child.start && end <= child.end) || // child has end byte - (offset < child.start && end > child.end) // child is between offset and end bytes - }) - - if (filteredLinks.length) { - // move stream position to the first node we're going to return data from - streamPosition = filteredLinks[0].start + if (buf.length) { + yield buf } - return pull( - once(filteredLinks), - paramap(async (children, cb) => { - try { - let results = [] - - for await (const result of await dag.getMany(children.map(child => child.link.cid))) { - const child = children[results.length] - - results.push({ - start: child.start, - end: child.end, - node: result, - size: child.size - }) - } - - cb(null, results) - } catch (err) { - cb(err) - } - }), - flatten() - ) + streamPosition += file.data.length } -} - -function extractData (requestedStart, requestedEnd) { - let streamPosition = -1 - return function getData ({ node, start, end }) { - let block + let childStart = streamPosition - if (Buffer.isBuffer(node)) { - block = node - } else { - try { - const file = UnixFS.unmarshal(node.data) + // work out which child nodes contain the requested data + for (let i = 0; i < node.links.length; i++) { + const childLink = node.links[i] + const childEnd = streamPosition + file.blockSizes[i] - if (!file.data) { - if (file.blockSizes.length) { - return - } + if ((start >= childStart && start < childEnd) || // child has offset byte + (end > childStart && end <= childEnd) || // child has end byte + (start < childStart && end > childEnd)) { // child is between offset and end bytes + const child = await ipld.get(childLink.cid) - return Buffer.alloc(0) - } + for await (const buf of emitBytes(ipld, child, start, end, streamPosition)) { + streamPosition += buf.length - block = file.data - } catch (err) { - throw new Error(`Failed to unmarshal node - ${err.message}`) + yield buf } } - if (block && block.length) { - if (streamPosition === -1) { - streamPosition = start - } - - const output = extractDataFromBlock(block, streamPosition, requestedStart, requestedEnd) - - streamPosition += block.length - - return output - } - - return Buffer.alloc(0) + streamPosition = childEnd + childStart = childEnd + 1 } } const fileContent = (cid, node, unixfs, path, resolve, ipld) => { return (options = {}) => { - return toIterator(streamBytes(ipld, node, unixfs.fileSize(), options)) + const fileSize = unixfs.fileSize() + + const { + offset, + length + } = validateOffsetAndLength(fileSize, options.offset, options.length) + + const start = offset + const end = offset + length + + return emitBytes(ipld, node, start, end) } } diff --git a/src/resolvers/unixfs-v1/content/raw.js b/src/resolvers/unixfs-v1/content/raw.js index 7e905de..0d973b2 100644 --- a/src/resolvers/unixfs-v1/content/raw.js +++ b/src/resolvers/unixfs-v1/content/raw.js @@ -1,42 +1,18 @@ 'use strict' -const errCode = require('err-code') const extractDataFromBlock = require('../../../utils/extract-data-from-block') -const toIterator = require('pull-stream-to-async-iterator') -const once = require('pull-stream/sources/once') +const validateOffsetAndLength = require('../../../utils/validate-offset-and-length') const rawContent = async (cid, node, unixfs, path, resolve, ipld) => { - return (options = {}) => { + return async function * (options = {}) { const size = node.length - let offset = options.offset - let length = options.length + const { + offset, + length + } = validateOffsetAndLength(size, options.offset, options.length) - if (offset < 0) { - throw errCode(new Error('Offset must be greater than or equal to 0'), 'EINVALIDPARAMS') - } - - if (offset > size) { - throw errCode(new Error('Offset must be less than the file size'), 'EINVALIDPARAMS') - } - - if (length < 0) { - throw errCode(new Error('Length must be greater than or equal to 0'), 'EINVALIDPARAMS') - } - - if (length === 0) { - return toIterator(once(Buffer.alloc(0))) - } - - if (!offset) { - offset = 0 - } - - if (!length || (offset + length > size)) { - length = size - offset - } - - return toIterator(once(extractDataFromBlock(unixfs.data, 0, offset, offset + length))) + yield extractDataFromBlock(unixfs.data, 0, offset, offset + length) } } diff --git a/src/utils/validate-offset-and-length.js b/src/utils/validate-offset-and-length.js new file mode 100644 index 0000000..1f207f5 --- /dev/null +++ b/src/utils/validate-offset-and-length.js @@ -0,0 +1,36 @@ +'use strict' + +const errCode = require('err-code') + +const validateOffsetAndLength = (size, offset, length) => { + if (!offset) { + offset = 0 + } + + if (offset < 0) { + throw errCode(new Error('Offset must be greater than or equal to 0'), 'EINVALIDPARAMS') + } + + if (offset > size) { + throw errCode(new Error('Offset must be less than the file size'), 'EINVALIDPARAMS') + } + + if (!length && length !== 0) { + length = size - offset + } + + if (length < 0) { + throw errCode(new Error('Length must be greater than or equal to 0'), 'EINVALIDPARAMS') + } + + if (offset + length > size) { + length = size - offset + } + + return { + offset, + length + } +} + +module.exports = validateOffsetAndLength diff --git a/test/exporter.spec.js b/test/exporter.spec.js index df57f4a..1d65f72 100644 --- a/test/exporter.spec.js +++ b/test/exporter.spec.js @@ -91,6 +91,31 @@ describe('exporter', () => { expect(data).to.deep.equal(Buffer.from([1, 2, 3])) } + async function createAndPersistNode (ipld, type, data, children) { + const file = new UnixFS(type, data ? Buffer.from(data) : undefined) + const links = [] + + for (let i = 0; i < children.length; i++) { + const child = children[i] + const leaf = UnixFS.unmarshal(child.node.data) + + file.addBlockSize(leaf.fileSize()) + + links.push(await DAGLink.create('', child.node.size, child.cid)) + } + + const node = await DAGNode.create(file.marshal(), links) + const cid = await ipld.put(node, mc.DAG_PB, { + cidVersion: 1, + hashAlg: mh.names['sha2-256'] + }) + + return { + node, + cid + } + } + before((done) => { inMemory(IPLD, (err, resolver) => { expect(err).to.not.exist() @@ -170,17 +195,37 @@ describe('exporter', () => { }) it('exports a small file with links', async () => { - const chunk = await dagPut({ content: randomBytes(100) }) - const result = await dagPut({ - content: Buffer.concat(await all(randomBytes(100))), - links: [ - await DAGLink.create('', chunk.node.size, chunk.cid) - ] + const content = Buffer.from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + const chunk1 = new UnixFS('raw', content.slice(0, 5)) + const chunkNode1 = await DAGNode.create(chunk1.marshal()) + const chunkCid1 = await ipld.put(chunkNode1, mc.DAG_PB, { + cidVersion: 0, + hashAlg: mh.names['sha2-256'] }) - const file = await exporter(result.cid, ipld) - const data = Buffer.concat(await all(file.content())) - expect(data).to.deep.equal(result.file.data) + const chunk2 = new UnixFS('raw', content.slice(5)) + const chunkNode2 = await DAGNode.create(chunk2.marshal()) + const chunkCid2 = await ipld.put(chunkNode2, mc.DAG_PB, { + cidVersion: 0, + hashAlg: mh.names['sha2-256'] + }) + + const file = new UnixFS('file') + file.addBlockSize(5) + file.addBlockSize(5) + + const fileNode = await DAGNode.create(file.marshal(), [ + await DAGLink.create('', chunkNode1.size, chunkCid1), + await DAGLink.create('', chunkNode2.size, chunkCid2) + ]) + const fileCid = await ipld.put(fileNode, mc.DAG_PB, { + cidVersion: 0, + hashAlg: mh.names['sha2-256'] + }) + + const exported = await exporter(fileCid, ipld) + const data = Buffer.concat(await all(exported.content())) + expect(data).to.deep.equal(content) }) it('exports a chunk of a small file with links', async () => { @@ -518,6 +563,24 @@ describe('exporter', () => { expect(data).to.deep.equal(Buffer.from([0])) }) + it('reads returns an empty buffer when offset is equal to the file size', async () => { + const data = await addAndReadTestFile({ + file: Buffer.from([0, 1, 2, 3]), + offset: 4 + }) + + expect(data).to.be.empty() + }) + + it('reads returns an empty buffer when length is zero', async () => { + const data = await addAndReadTestFile({ + file: Buffer.from([0, 1, 2, 3]), + length: 0 + }) + + expect(data).to.be.empty() + }) + it('errors when reading bytes with a negative length', async () => { try { await addAndReadTestFile({ @@ -531,6 +594,18 @@ describe('exporter', () => { } }) + it('errors when reading bytes that start after the file ends', async () => { + try { + await addAndReadTestFile({ + file: Buffer.from([0, 1, 2, 3, 4]), + offset: 200 + }) + } catch (err) { + expect(err.message).to.contain('Offset must be less than the file size') + expect(err.code).to.equal('EINVALIDPARAMS') + } + }) + it('reads bytes with an offset and a length', async () => { const data = await addAndReadTestFile({ file: Buffer.from([0, 1, 2, 3, 4]), @@ -703,28 +778,3 @@ describe('exporter', () => { expect(data).to.deep.equal(smallFile) }) }) - -async function createAndPersistNode (ipld, type, data, children) { - const file = new UnixFS(type, data ? Buffer.from(data) : undefined) - const links = [] - - for (let i = 0; i < children.length; i++) { - const child = children[i] - const leaf = UnixFS.unmarshal(child.node.data) - - file.addBlockSize(leaf.fileSize()) - - links.push(await DAGLink.create('', child.node.size, child.cid)) - } - - const node = await DAGNode.create(file.marshal(), links) - const cid = await ipld.put(node, mc.DAG_PB, { - cidVersion: 1, - hashAlg: mh.names['sha2-256'] - }) - - return { - node, - cid - } -} From 468cdfb59da563f2a6185bf5001a71b556f457df Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 26 Apr 2019 10:20:23 +0100 Subject: [PATCH 05/11] test: increase test coverage --- package.json | 3 +- src/resolvers/raw.js | 12 ++-- src/resolvers/unixfs-v1/content/file.js | 3 +- src/resolvers/unixfs-v1/index.js | 2 +- test/exporter.spec.js | 74 +++++++++++++++++++++++++ 5 files changed, 83 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index ecf43dd..b5d6bea 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "release": "aegir release", "release-minor": "aegir release --type minor", "release-major": "aegir release --type major", - "coverage": "aegir coverage", + "coverage": "nyc -s npm run test:node && nyc report --reporter=html", "dep-check": "aegir dep-check" }, "repository": { @@ -49,6 +49,7 @@ "ipld-in-memory": "^2.0.0", "multicodec": "~0.5.1", "multihashes": "~0.4.14", + "nyc": "^14.0.0", "sinon": "^7.1.0" }, "dependencies": { diff --git a/src/resolvers/raw.js b/src/resolvers/raw.js index ae7d821..dee845a 100644 --- a/src/resolvers/raw.js +++ b/src/resolvers/raw.js @@ -16,14 +16,10 @@ const rawContent = (node) => { } const resolve = async (cid, name, path, toResolve, resolve, ipld) => { - const node = await ipld.get(cid) - - if (!Buffer.isBuffer(node)) { - throw errCode(new Error(`'${cid.codec}' node ${cid.toBaseEncodedString()} was not a buffer`), 'ENOBUF') - } + const buf = await ipld.get(cid) if (toResolve.length) { - throw errCode(new Error(`No link named ${path} found in raw node ${cid.toBaseEncodedString()}`), 'ENOLINK') + throw errCode(new Error(`No link named ${path} found in raw node ${cid.toBaseEncodedString()}`), 'ENOTFOUND') } return { @@ -31,8 +27,8 @@ const resolve = async (cid, name, path, toResolve, resolve, ipld) => { name, path, cid, - node, - content: rawContent(node) + node: buf, + content: rawContent(buf) } } } diff --git a/src/resolvers/unixfs-v1/content/file.js b/src/resolvers/unixfs-v1/content/file.js index 421ec23..500c4c7 100644 --- a/src/resolvers/unixfs-v1/content/file.js +++ b/src/resolvers/unixfs-v1/content/file.js @@ -3,6 +3,7 @@ const extractDataFromBlock = require('../../../utils/extract-data-from-block') const validateOffsetAndLength = require('../../../utils/validate-offset-and-length') const UnixFS = require('ipfs-unixfs') +const errCode = require('err-code') async function * emitBytes (ipld, node, start, end, streamPosition = 0) { // a `raw` node @@ -23,7 +24,7 @@ async function * emitBytes (ipld, node, start, end, streamPosition = 0) { try { file = UnixFS.unmarshal(node.data) } catch (err) { - throw err + throw errCode(err, 'ENOTUNIXFS') } // might be a unixfs `raw` node or have data on intermediate nodes diff --git a/src/resolvers/unixfs-v1/index.js b/src/resolvers/unixfs-v1/index.js index 2d356f4..3adf2b7 100644 --- a/src/resolvers/unixfs-v1/index.js +++ b/src/resolvers/unixfs-v1/index.js @@ -32,7 +32,7 @@ const unixFsResolver = async (cid, name, path, toResolve, resolve, ipld) => { unixfs = UnixFS.unmarshal(node.data) } catch (err) { // non-UnixFS dag-pb node? It could happen. - throw err + throw errCode(err, 'ENOTUNIXFS') } if (!path) { diff --git a/test/exporter.spec.js b/test/exporter.spec.js index 1d65f72..35f0c68 100644 --- a/test/exporter.spec.js +++ b/test/exporter.spec.js @@ -777,4 +777,78 @@ describe('exporter', () => { const data = Buffer.concat(await all(file.content())) expect(data).to.deep.equal(smallFile) }) + + it('errors when exporting a non-existent key from a cbor node', async () => { + const cborNodeCid = await ipld.put({ + foo: 'bar' + }, mc.DAG_CBOR) + + try { + await exporter(`${cborNodeCid.toBaseEncodedString()}/baz`, ipld) + } catch (err) { + expect(err.code).to.equal('ENOPROP') + } + }) + + it('exports a cbor node', async () => { + const node = { + foo: 'bar' + } + + const cborNodeCid = await ipld.put(node, mc.DAG_CBOR) + const exported = await exporter(`${cborNodeCid.toBaseEncodedString()}`, ipld) + + expect(exported.node).to.deep.equal(node) + }) + + it('errors when exporting a node with no resolver', async () => { + const cid = new CID(1, 'git-raw', new CID('zdj7WkRPAX9o9nb9zPbXzwG7JEs78uyhwbUs8JSUayB98DWWY').multihash) + + try { + await exporter(`${cid.toBaseEncodedString()}`, ipld) + } catch (err) { + expect(err.code).to.equal('ENORESOLVER') + } + }) + + it('errors if we try to export links from inside a raw node', async () => { + const cid = await ipld.put(Buffer.from([0, 1, 2, 3, 4]), mc.RAW) + + try { + await exporter(`${cid.toBaseEncodedString()}/lol`, ipld) + } catch (err) { + expect(err.code).to.equal('ENOTFOUND') + } + }) + + it('errors we export a non-unixfs dag-pb node', async () => { + const cid = await ipld.put(await DAGNode.create(Buffer.from([0, 1, 2, 3, 4])), mc.DAG_PB) + + try { + await exporter(cid, ipld) + } catch (err) { + expect(err.code).to.equal('ENOTUNIXFS') + } + }) + + it('errors we export a unixfs node that has a non-unixfs/dag-pb child', async () => { + const cborNodeCid = await ipld.put({ + foo: 'bar' + }, mc.DAG_CBOR) + + const file = new UnixFS('file') + file.addBlockSize(100) + + const cid = await ipld.put(await DAGNode.create(file.marshal(), [ + await DAGLink.create('', 100, cborNodeCid) + ]), mc.DAG_PB) + + const exported = await exporter(cid, ipld) + + try { + await all(exported.content()) + } catch (err) { + expect(err.code).to.equal('ENOTUNIXFS') + } + }) }) From 220de8ad134eb594e3ede7824b7dc1ac0f567ce3 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 26 Apr 2019 11:49:49 +0100 Subject: [PATCH 06/11] chore: remove uncessary await --- src/resolvers/raw.js | 2 +- src/resolvers/unixfs-v1/content/raw.js | 4 ++-- test/helpers/create-shard.js | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/resolvers/raw.js b/src/resolvers/raw.js index dee845a..145ef8e 100644 --- a/src/resolvers/raw.js +++ b/src/resolvers/raw.js @@ -5,7 +5,7 @@ const extractDataFromBlock = require('../utils/extract-data-from-block') const validateOffsetAndLength = require('../utils/validate-offset-and-length') const rawContent = (node) => { - return async function * (options = {}) { + return function * (options = {}) { const { offset, length diff --git a/src/resolvers/unixfs-v1/content/raw.js b/src/resolvers/unixfs-v1/content/raw.js index 0d973b2..af41a68 100644 --- a/src/resolvers/unixfs-v1/content/raw.js +++ b/src/resolvers/unixfs-v1/content/raw.js @@ -3,8 +3,8 @@ const extractDataFromBlock = require('../../../utils/extract-data-from-block') const validateOffsetAndLength = require('../../../utils/validate-offset-and-length') -const rawContent = async (cid, node, unixfs, path, resolve, ipld) => { - return async function * (options = {}) { +const rawContent = (cid, node, unixfs, path, resolve, ipld) => { + return function * (options = {}) { const size = node.length const { diff --git a/test/helpers/create-shard.js b/test/helpers/create-shard.js index 96cd9e9..8cea770 100644 --- a/test/helpers/create-shard.js +++ b/test/helpers/create-shard.js @@ -4,11 +4,11 @@ const importer = require('ipfs-unixfs-importer') const SHARD_SPLIT_THRESHOLD = 10 -const createShard = async (numFiles, ipld) => { +const createShard = (numFiles, ipld) => { return createShardWithFileNames(numFiles, (index) => `file-${index}`, ipld) } -const createShardWithFileNames = async (numFiles, fileName, ipld) => { +const createShardWithFileNames = (numFiles, fileName, ipld) => { const files = new Array(numFiles).fill(0).map((_, index) => ({ path: fileName(index), content: Buffer.from([0, 1, 2, 3, 4, index]) From 3679e1d1bd60fa8ca532b59b97d07b4781b813af Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 14 May 2019 16:21:22 +0100 Subject: [PATCH 07/11] feat: add export depth and recursive exports --- README.md | 17 ++++++- src/index.js | 28 ++++++++++- src/resolvers/dag-cbor.js | 8 ++-- src/resolvers/index.js | 4 +- src/resolvers/raw.js | 5 +- src/resolvers/unixfs-v1/content/directory.js | 4 +- src/resolvers/unixfs-v1/content/file.js | 2 +- .../content/hamt-sharded-directory.js | 6 +-- src/resolvers/unixfs-v1/content/raw.js | 2 +- src/resolvers/unixfs-v1/index.js | 13 +++--- test/exporter-subtree.spec.js | 2 +- test/exporter.spec.js | 46 +++++++++++++++++++ 12 files changed, 114 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 73181c4..fd724f6 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ - [CBOR entries](#cbor-entries) - [`entry.content({ offset, length })`](#entrycontent-offset-length) - [`exporter.path(cid, ipld)`](#exporterpathcid-ipld) + - [`exporter.recursive(cid, ipld)`](#exporterrecursivecid-ipld) - [Contribute](#contribute) - [License](#license) @@ -204,13 +205,27 @@ for await (const entry of dir.content({ ```javascript const entries = [] -for await (const entry of exporter('Qmfoo/foo/bar/baz.txt', ipld)) { +for await (const entry of exporter.path('Qmfoo/foo/bar/baz.txt', ipld)) { entries.push(entry) } // entries contains 4x `entry` objects ``` +### `exporter.recursive(cid, ipld)` + +`exporter.recursive` will return an async iterator that yields all entries beneath a given CID or IPFS path, as well as the containing directory. + +```javascript +const entries = [] + +for await (const child of exporter.recursive('Qmfoo/foo/bar', ipld)) { + entries.push(entry) +} + +// entries contains all children of the `Qmfoo/foo/bar` directory and it's children +``` + [dag API]: https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md [ipld-resolver instance]: https://github.com/ipld/js-ipld-resolver [UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs diff --git a/src/index.js b/src/index.js index c85471b..54a20ad 100644 --- a/src/index.js +++ b/src/index.js @@ -51,9 +51,10 @@ const walkPath = async function * (path, ipld) { } = cidAndRest(path) let name = cid.toBaseEncodedString() let entryPath = name + const startingDepth = toResolve.length while (true) { - const result = await resolve(cid, name, entryPath, toResolve, ipld) + const result = await resolve(cid, name, entryPath, toResolve, startingDepth, ipld) if (!result.entry && !result.next) { throw errCode(new Error(`Could not resolve ${path}`), 'ENOTFOUND') @@ -79,5 +80,30 @@ const exporter = (path, ipld) => { return last(walkPath(path, ipld)) } +const recursive = async function * (path, ipld) { + const node = await exporter(path, ipld) + + yield node + + if (node.unixfs && node.unixfs.type.includes('dir')) { + for await (const child of recurse(node)) { + yield child + } + } + + async function * recurse (node) { + for await (const file of node.content()) { + yield file + + if (file.unixfs.type.includes('dir')) { + for await (const subFile of recurse(file)) { + yield subFile + } + } + } + } +} + module.exports = exporter module.exports.path = walkPath +module.exports.recursive = recursive diff --git a/src/resolvers/dag-cbor.js b/src/resolvers/dag-cbor.js index fcc851c..247a2a6 100644 --- a/src/resolvers/dag-cbor.js +++ b/src/resolvers/dag-cbor.js @@ -3,7 +3,7 @@ const CID = require('cids') const errCode = require('err-code') -const resolve = async (cid, name, path, toResolve, resolve, ipld) => { +const resolve = async (cid, name, path, toResolve, resolve, depth, ipld) => { let node = await ipld.get(cid) let subObject = node let subPath = path @@ -22,7 +22,8 @@ const resolve = async (cid, name, path, toResolve, resolve, ipld) => { name, path, cid, - node + node, + depth }, next: { cid: subObject[prop], @@ -45,7 +46,8 @@ const resolve = async (cid, name, path, toResolve, resolve, ipld) => { name, path, cid, - node + node, + depth } } } diff --git a/src/resolvers/index.js b/src/resolvers/index.js index a80ef7e..bb35964 100644 --- a/src/resolvers/index.js +++ b/src/resolvers/index.js @@ -8,14 +8,14 @@ const resolvers = { 'dag-cbor': require('./dag-cbor') } -const resolve = (cid, name, path, toResolve, ipld) => { +const resolve = (cid, name, path, toResolve, depth, ipld) => { const resolver = resolvers[cid.codec] if (!resolver) { throw errCode(new Error(`No resolver for codec ${cid.codec}`), 'ENORESOLVER') } - return resolver(cid, name, path, toResolve, resolve, ipld) + return resolver(cid, name, path, toResolve, resolve, depth, ipld) } module.exports = resolve diff --git a/src/resolvers/raw.js b/src/resolvers/raw.js index 145ef8e..7d75c74 100644 --- a/src/resolvers/raw.js +++ b/src/resolvers/raw.js @@ -15,7 +15,7 @@ const rawContent = (node) => { } } -const resolve = async (cid, name, path, toResolve, resolve, ipld) => { +const resolve = async (cid, name, path, toResolve, resolve, depth, ipld) => { const buf = await ipld.get(cid) if (toResolve.length) { @@ -28,7 +28,8 @@ const resolve = async (cid, name, path, toResolve, resolve, ipld) => { path, cid, node: buf, - content: rawContent(buf) + content: rawContent(buf), + depth } } } diff --git a/src/resolvers/unixfs-v1/content/directory.js b/src/resolvers/unixfs-v1/content/directory.js index 0e19703..9684dee 100644 --- a/src/resolvers/unixfs-v1/content/directory.js +++ b/src/resolvers/unixfs-v1/content/directory.js @@ -1,13 +1,13 @@ 'use strict' -const directoryContent = (cid, node, unixfs, path, resolve, ipld) => { +const directoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => { return async function * (options = {}) { const offset = options.offset || 0 const length = options.length || node.links.length const links = node.links.slice(offset, length) for (const link of links) { - const result = await resolve(link.cid, link.name, `${path}/${link.name}`, [], ipld) + const result = await resolve(link.cid, link.name, `${path}/${link.name}`, [], depth + 1, ipld) yield result.entry } diff --git a/src/resolvers/unixfs-v1/content/file.js b/src/resolvers/unixfs-v1/content/file.js index 500c4c7..ce72f4a 100644 --- a/src/resolvers/unixfs-v1/content/file.js +++ b/src/resolvers/unixfs-v1/content/file.js @@ -64,7 +64,7 @@ async function * emitBytes (ipld, node, start, end, streamPosition = 0) { } } -const fileContent = (cid, node, unixfs, path, resolve, ipld) => { +const fileContent = (cid, node, unixfs, path, resolve, depth, ipld) => { return (options = {}) => { const fileSize = unixfs.fileSize() diff --git a/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js b/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js index 1c9d3ab..84b6544 100644 --- a/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js +++ b/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js @@ -1,6 +1,6 @@ 'use strict' -const hamtShardedDirectoryContent = (cid, node, unixfs, path, resolve, ipld) => { +const hamtShardedDirectoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => { return async function * (options = {}) { const links = node.links @@ -8,14 +8,14 @@ const hamtShardedDirectoryContent = (cid, node, unixfs, path, resolve, ipld) => const name = link.name.substring(2) if (name) { - const result = await resolve(link.cid, name, `${path}/${name}`, [], ipld) + const result = await resolve(link.cid, name, `${path}/${name}`, [], depth + 1, ipld) yield result.entry } else { // descend into subshard node = await ipld.get(link.cid) - for await (const file of hamtShardedDirectoryContent(link.cid, node, null, path, resolve, ipld)(options)) { + for await (const file of hamtShardedDirectoryContent(link.cid, node, null, path, resolve, depth, ipld)(options)) { yield file } } diff --git a/src/resolvers/unixfs-v1/content/raw.js b/src/resolvers/unixfs-v1/content/raw.js index af41a68..cbef4af 100644 --- a/src/resolvers/unixfs-v1/content/raw.js +++ b/src/resolvers/unixfs-v1/content/raw.js @@ -3,7 +3,7 @@ const extractDataFromBlock = require('../../../utils/extract-data-from-block') const validateOffsetAndLength = require('../../../utils/validate-offset-and-length') -const rawContent = (cid, node, unixfs, path, resolve, ipld) => { +const rawContent = (cid, node, unixfs, path, resolve, depth, ipld) => { return function * (options = {}) { const size = node.length diff --git a/src/resolvers/unixfs-v1/index.js b/src/resolvers/unixfs-v1/index.js index 3adf2b7..1bcf394 100644 --- a/src/resolvers/unixfs-v1/index.js +++ b/src/resolvers/unixfs-v1/index.js @@ -15,11 +15,11 @@ const contentExporters = { file: require('./content/file'), directory: require('./content/directory'), 'hamt-sharded-directory': require('./content/hamt-sharded-directory'), - metadata: (cid, node, unixfs, ipld) => {}, - symlink: (cid, node, unixfs, ipld) => {} + metadata: (cid, node, unixfs, path, resolve, depth, ipld) => {}, + symlink: (cid, node, unixfs, path, resolve, depth, ipld) => {} } -const unixFsResolver = async (cid, name, path, toResolve, resolve, ipld) => { +const unixFsResolver = async (cid, name, path, toResolve, resolve, depth, ipld) => { const node = await ipld.get(cid) let unixfs let next @@ -50,7 +50,7 @@ const unixFsResolver = async (cid, name, path, toResolve, resolve, ipld) => { } if (!linkCid) { - throw errCode(new Error(`No link named ${toResolve} found in node ${cid.toBaseEncodedString()}`), 'ENOLINK') + throw errCode(new Error(`file does not exist`), 'ERR_NOT_FOUND') } // remove the path component we have resolved @@ -71,8 +71,9 @@ const unixFsResolver = async (cid, name, path, toResolve, resolve, ipld) => { path, cid, node, - content: contentExporters[unixfs.type](cid, node, unixfs, path, resolve, ipld), - unixfs + content: contentExporters[unixfs.type](cid, node, unixfs, path, resolve, depth, ipld), + unixfs, + depth }, next } diff --git a/test/exporter-subtree.spec.js b/test/exporter-subtree.spec.js index a7f66cb..2d1e37b 100644 --- a/test/exporter-subtree.spec.js +++ b/test/exporter-subtree.spec.js @@ -85,7 +85,7 @@ describe('exporter subtree', () => { try { await exporter(`${imported.cid.toBaseEncodedString()}/doesnotexist`, ipld) } catch (err) { - expect(err.code).to.equal('ENOLINK') + expect(err.code).to.equal('ERR_NOT_FOUND') } }) diff --git a/test/exporter.spec.js b/test/exporter.spec.js index 35f0c68..519e8fa 100644 --- a/test/exporter.spec.js +++ b/test/exporter.spec.js @@ -851,4 +851,50 @@ describe('exporter', () => { expect(err.code).to.equal('ENOTUNIXFS') } }) + + it('exports a node with depth', async () => { + const imported = await all(importer([{ + path: '/foo/bar/baz.txt', + content: Buffer.from('hello world') + }], ipld)) + + const exported = await exporter(imported[0].cid, ipld) + + expect(exported.depth).to.equal(0) + }) + + it('exports a node recursively with depth', async () => { + const dir = await last(importer([{ + path: '/foo/bar/baz.txt', + content: Buffer.from('hello world') + }, { + path: '/foo/qux.txt', + content: Buffer.from('hello world') + }, { + path: '/foo/bar/quux.txt', + content: Buffer.from('hello world') + }], ipld)) + + const exported = await all(exporter.recursive(dir.cid, ipld)) + const dirCid = dir.cid.toBaseEncodedString() + + expect(exported[0].depth).to.equal(0) + expect(exported[0].name).to.equal(dirCid) + + expect(exported[1].depth).to.equal(1) + expect(exported[1].name).to.equal('bar') + expect(exported[1].path).to.equal(`${dirCid}/bar`) + + expect(exported[2].depth).to.equal(2) + expect(exported[2].name).to.equal('baz.txt') + expect(exported[2].path).to.equal(`${dirCid}/bar/baz.txt`) + + expect(exported[3].depth).to.equal(2) + expect(exported[3].name).to.equal('quux.txt') + expect(exported[3].path).to.equal(`${dirCid}/bar/quux.txt`) + + expect(exported[4].depth).to.equal(1) + expect(exported[4].name).to.equal('qux.txt') + expect(exported[4].path).to.equal(`${dirCid}/qux.txt`) + }) }) From ef5499b3b35448d15ddd33d89306ee84f3b39a97 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 May 2019 10:02:43 +0100 Subject: [PATCH 08/11] chore: address PR comments --- README.md | 20 +++++++++++++------- package.json | 8 ++++---- src/resolvers/raw.js | 4 ++-- test/helpers/dag-pb.js | 30 ------------------------------ 4 files changed, 19 insertions(+), 43 deletions(-) delete mode 100644 test/helpers/dag-pb.js diff --git a/README.md b/README.md index fd724f6..f8c491d 100644 --- a/README.md +++ b/README.md @@ -111,19 +111,23 @@ Entries with a `dag-pb` codec `CID` return UnixFS V1 entries: } ``` -If the entry is a file, `entry.content()` returns an async iterator that emits buffers containing the file content: +If the entry is a file, `entry.content()` returns an async iterator that yields one or more buffers containing the file content: ```javascript -for await (const chunk of entry.content()) { - // chunk is a Buffer +if (entry.unixfs.type === 'file') { + for await (const chunk of entry.content()) { + // chunk is a Buffer + } } ``` If the entry is a directory or hamt shard, `entry.content()` returns further `entry` objects: ```javascript -for await (const entry of dir.content()) { - console.info(entry.name) +if (entry.unixfs.type.includes('directory')) { // can be 'directory' or 'hamt-sharded-directory' + for await (const entry of dir.content()) { + console.info(entry.name) + } } ``` @@ -141,7 +145,7 @@ Entries with a `raw` codec `CID` return raw entries: } ``` -`entry.content()` returns an async iterator that emits buffers containing the node content: +`entry.content()` returns an async iterator that yields a buffer containing the node content: ```javascript for await (const chunk of entry.content()) { @@ -149,6 +153,8 @@ for await (const chunk of entry.content()) { } ``` +Unless you an options object containing `offset` and `length` keys as an argument to `entry.content()`, `chunk` will be equal to `entry.node`. + #### CBOR entries Entries with a `dag-cbor` codec `CID` return JavaScript object entries: @@ -183,7 +189,7 @@ for await (const chunk of entry.content({ const data = Buffer.concat(bufs) ``` -If `entry` is a directory or hamt shard, passing `offset` and/or `length` to `entry.content()` will limit the number of files return from the directory. +If `entry` is a directory or hamt shard, passing `offset` and/or `length` to `entry.content()` will limit the number of files returned from the directory. ```javascript const entries = [] diff --git a/package.json b/package.json index b5d6bea..4330041 100644 --- a/package.json +++ b/package.json @@ -44,8 +44,8 @@ "chai": "^4.2.0", "detect-node": "^2.0.4", "dirty-chai": "^2.0.1", - "ipld": "~0.22.0", - "ipld-dag-pb": "~0.15.2", + "ipld": "~0.24.0", + "ipld-dag-pb": "~0.17.0", "ipld-in-memory": "^2.0.0", "multicodec": "~0.5.1", "multihashes": "~0.4.14", @@ -54,11 +54,11 @@ }, "dependencies": { "async-iterator-last": "0.0.2", - "cids": "~0.5.5", + "cids": "~0.7.1", "err-code": "^1.1.2", "hamt-sharding": "0.0.2", "ipfs-unixfs": "~0.1.16", - "ipfs-unixfs-importer": "ipfs/js-ipfs-unixfs-importer#async-await", + "ipfs-unixfs-importer": "~0.38.5", "promisify-es6": "^1.0.3" }, "contributors": [ diff --git a/src/resolvers/raw.js b/src/resolvers/raw.js index 7d75c74..dbb95bf 100644 --- a/src/resolvers/raw.js +++ b/src/resolvers/raw.js @@ -16,12 +16,12 @@ const rawContent = (node) => { } const resolve = async (cid, name, path, toResolve, resolve, depth, ipld) => { - const buf = await ipld.get(cid) - if (toResolve.length) { throw errCode(new Error(`No link named ${path} found in raw node ${cid.toBaseEncodedString()}`), 'ENOTFOUND') } + const buf = await ipld.get(cid) + return { entry: { name, diff --git a/test/helpers/dag-pb.js b/test/helpers/dag-pb.js deleted file mode 100644 index 947422b..0000000 --- a/test/helpers/dag-pb.js +++ /dev/null @@ -1,30 +0,0 @@ -'use strict' - -const { - DAGNode, - DAGLink -} = require('ipld-dag-pb') -const promisify = require('promisify-es6') - -const node = { - create: promisify(DAGNode.create, { - context: DAGNode - }), - addLink: promisify(DAGNode.addLink, { - context: DAGNode - }), - rmLink: promisify(DAGNode.rmLink, { - context: DAGNode - }) -} - -const link = { - create: promisify(DAGLink.create, { - context: DAGLink - }) -} - -module.exports = { - DAGNode: node, - DAGLink: link -} From 747a6648efaf7e12fdd79c96dd53fca53e976398 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 May 2019 20:55:33 +0100 Subject: [PATCH 09/11] chore: update ipld formats --- package.json | 13 +++++------ src/resolvers/unixfs-v1/content/directory.js | 6 ++--- src/resolvers/unixfs-v1/content/file.js | 8 +++---- .../content/hamt-sharded-directory.js | 10 ++++----- src/resolvers/unixfs-v1/index.js | 6 ++--- src/utils/find-cid-in-shard.js | 22 +++++++++---------- test/exporter-sharded.spec.js | 12 +++++----- test/exporter.spec.js | 20 ++++++++--------- 8 files changed, 48 insertions(+), 49 deletions(-) diff --git a/package.json b/package.json index 4330041..676f11e 100644 --- a/package.json +++ b/package.json @@ -38,9 +38,9 @@ "homepage": "https://github.com/ipfs/js-ipfs-unixfs-exporter#readme", "devDependencies": { "aegir": "^18.0.2", - "async-iterator-all": "0.0.2", - "async-iterator-buffer-stream": "0.0.1", - "async-iterator-first": "0.0.2", + "async-iterator-all": "^1.0.0", + "async-iterator-buffer-stream": "^1.0.0", + "async-iterator-first": "^1.0.0", "chai": "^4.2.0", "detect-node": "^2.0.4", "dirty-chai": "^2.0.1", @@ -53,13 +53,12 @@ "sinon": "^7.1.0" }, "dependencies": { - "async-iterator-last": "0.0.2", + "async-iterator-last": "^1.0.0", "cids": "~0.7.1", "err-code": "^1.1.2", - "hamt-sharding": "0.0.2", + "hamt-sharding": "~0.0.2", "ipfs-unixfs": "~0.1.16", - "ipfs-unixfs-importer": "~0.38.5", - "promisify-es6": "^1.0.3" + "ipfs-unixfs-importer": "~0.38.5" }, "contributors": [ "Alan Shaw ", diff --git a/src/resolvers/unixfs-v1/content/directory.js b/src/resolvers/unixfs-v1/content/directory.js index 9684dee..df909e1 100644 --- a/src/resolvers/unixfs-v1/content/directory.js +++ b/src/resolvers/unixfs-v1/content/directory.js @@ -3,11 +3,11 @@ const directoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => { return async function * (options = {}) { const offset = options.offset || 0 - const length = options.length || node.links.length - const links = node.links.slice(offset, length) + const length = options.length || node.Links.length + const links = node.Links.slice(offset, length) for (const link of links) { - const result = await resolve(link.cid, link.name, `${path}/${link.name}`, [], depth + 1, ipld) + const result = await resolve(link.Hash, link.Name, `${path}/${link.Name}`, [], depth + 1, ipld) yield result.entry } diff --git a/src/resolvers/unixfs-v1/content/file.js b/src/resolvers/unixfs-v1/content/file.js index ce72f4a..51333db 100644 --- a/src/resolvers/unixfs-v1/content/file.js +++ b/src/resolvers/unixfs-v1/content/file.js @@ -22,7 +22,7 @@ async function * emitBytes (ipld, node, start, end, streamPosition = 0) { let file try { - file = UnixFS.unmarshal(node.data) + file = UnixFS.unmarshal(node.Data) } catch (err) { throw errCode(err, 'ENOTUNIXFS') } @@ -43,14 +43,14 @@ async function * emitBytes (ipld, node, start, end, streamPosition = 0) { let childStart = streamPosition // work out which child nodes contain the requested data - for (let i = 0; i < node.links.length; i++) { - const childLink = node.links[i] + for (let i = 0; i < node.Links.length; i++) { + const childLink = node.Links[i] const childEnd = streamPosition + file.blockSizes[i] if ((start >= childStart && start < childEnd) || // child has offset byte (end > childStart && end <= childEnd) || // child has end byte (start < childStart && end > childEnd)) { // child is between offset and end bytes - const child = await ipld.get(childLink.cid) + const child = await ipld.get(childLink.Hash) for await (const buf of emitBytes(ipld, child, start, end, streamPosition)) { streamPosition += buf.length diff --git a/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js b/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js index 84b6544..29f22fc 100644 --- a/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js +++ b/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js @@ -2,20 +2,20 @@ const hamtShardedDirectoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => { return async function * (options = {}) { - const links = node.links + const links = node.Links for (const link of links) { - const name = link.name.substring(2) + const name = link.Name.substring(2) if (name) { - const result = await resolve(link.cid, name, `${path}/${name}`, [], depth + 1, ipld) + const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, ipld) yield result.entry } else { // descend into subshard - node = await ipld.get(link.cid) + node = await ipld.get(link.Hash) - for await (const file of hamtShardedDirectoryContent(link.cid, node, null, path, resolve, depth, ipld)(options)) { + for await (const file of hamtShardedDirectoryContent(link.Hash, node, null, path, resolve, depth, ipld)(options)) { yield file } } diff --git a/src/resolvers/unixfs-v1/index.js b/src/resolvers/unixfs-v1/index.js index 1bcf394..b56c66c 100644 --- a/src/resolvers/unixfs-v1/index.js +++ b/src/resolvers/unixfs-v1/index.js @@ -5,9 +5,9 @@ const UnixFS = require('ipfs-unixfs') const findShardCid = require('../../utils/find-cid-in-shard') const findLinkCid = (node, name) => { - const link = node.links.find(link => link.name === name) + const link = node.Links.find(link => link.Name === name) - return link && link.cid + return link && link.Hash } const contentExporters = { @@ -29,7 +29,7 @@ const unixFsResolver = async (cid, name, path, toResolve, resolve, depth, ipld) } try { - unixfs = UnixFS.unmarshal(node.data) + unixfs = UnixFS.unmarshal(node.Data) } catch (err) { // non-UnixFS dag-pb node? It could happen. throw errCode(err, 'ENOTUNIXFS') diff --git a/src/utils/find-cid-in-shard.js b/src/utils/find-cid-in-shard.js index 789b5b8..11d6c09 100644 --- a/src/utils/find-cid-in-shard.js +++ b/src/utils/find-cid-in-shard.js @@ -1,20 +1,20 @@ 'use strict' const Bucket = require('hamt-sharding/src/bucket') -const DirSharded = require('ipfs-unixfs-importer/src/importer/dir-sharded') +const DirSharded = require('ipfs-unixfs-importer/src/dir-sharded') const addLinksToHamtBucket = (links, bucket, rootBucket) => { return Promise.all( links.map(link => { - if (link.name.length === 2) { - const pos = parseInt(link.name, 16) + if (link.Name.length === 2) { + const pos = parseInt(link.Name, 16) return bucket._putObjectAt(pos, new Bucket({ hashFn: DirSharded.hashFn }, bucket, pos)) } - return rootBucket.put(link.name.substring(2), true) + return rootBucket.put(link.Name.substring(2), true) }) ) } @@ -54,7 +54,7 @@ const findShardCid = async (node, name, ipld, context) => { context.lastBucket = context.rootBucket } - await addLinksToHamtBucket(node.links, context.lastBucket, context.rootBucket) + await addLinksToHamtBucket(node.Links, context.lastBucket, context.rootBucket) const position = await context.rootBucket._findNewBucketAndPos(name) let prefix = toPrefix(position.pos) @@ -66,9 +66,9 @@ const findShardCid = async (node, name, ipld, context) => { prefix = toPrefix(context.lastBucket._posAtParent) } - const link = node.links.find(link => { - const entryPrefix = link.name.substring(0, 2) - const entryName = link.name.substring(2) + const link = node.Links.find(link => { + const entryPrefix = link.Name.substring(0, 2) + const entryName = link.Name.substring(2) if (entryPrefix !== prefix) { // not the entry or subshard we're looking for @@ -87,13 +87,13 @@ const findShardCid = async (node, name, ipld, context) => { return null } - if (link.name.substring(2) === name) { - return link.cid + if (link.Name.substring(2) === name) { + return link.Hash } context.hamtDepth++ - node = await ipld.get(link.cid) + node = await ipld.get(link.Hash) return findShardCid(node, name, ipld, context) } diff --git a/test/exporter-sharded.spec.js b/test/exporter-sharded.spec.js index e6bca31..fcfab68 100644 --- a/test/exporter-sharded.spec.js +++ b/test/exporter-sharded.spec.js @@ -17,7 +17,7 @@ const importer = require('ipfs-unixfs-importer') const { DAGLink, DAGNode -} = require('./helpers/dag-pb') +} = require('ipld-dag-pb') const SHARD_SPLIT_THRESHOLD = 10 @@ -42,7 +42,7 @@ describe('exporter sharded', function () { const createShardWithFiles = async (files) => { return (await last(importer(files, ipld, { shardSplitThreshold: SHARD_SPLIT_THRESHOLD, - wrap: true + wrapWithDirectory: true }))).cid } @@ -69,7 +69,7 @@ describe('exporter sharded', function () { path, content: files[path].content })), ipld, { - wrap: true, + wrapWithDirectory: true, shardSplitThreshold: SHARD_SPLIT_THRESHOLD })) @@ -81,7 +81,7 @@ describe('exporter sharded', function () { }) const dir = await ipld.get(dirCid) - const dirMetadata = UnixFS.unmarshal(dir.data) + const dirMetadata = UnixFS.unmarshal(dir.Data) expect(dirMetadata.type).to.equal('hamt-sharded-directory') @@ -191,7 +191,7 @@ describe('exporter sharded', function () { const dirCid = await createShard(15) const node = await DAGNode.create(new UnixFS('directory').marshal(), [ - await DAGLink.create('shard', 5, dirCid) + new DAGLink('shard', 5, dirCid) ]) const nodeCid = await ipld.put(node, mc.DAG_PB, { cidVersion: 0, @@ -199,7 +199,7 @@ describe('exporter sharded', function () { }) const shardNode = await DAGNode.create(new UnixFS('hamt-sharded-directory').marshal(), [ - await DAGLink.create('75normal-dir', 5, nodeCid) + new DAGLink('75normal-dir', 5, nodeCid) ]) const shardNodeCid = await ipld.put(shardNode, mc.DAG_PB, { cidVersion: 1, diff --git a/test/exporter.spec.js b/test/exporter.spec.js index 519e8fa..ae87120 100644 --- a/test/exporter.spec.js +++ b/test/exporter.spec.js @@ -11,7 +11,7 @@ const CID = require('cids') const { DAGNode, DAGLink -} = require('./helpers/dag-pb') +} = require('ipld-dag-pb') const mh = require('multihashes') const mc = require('multicodec') const exporter = require('../src') @@ -97,11 +97,11 @@ describe('exporter', () => { for (let i = 0; i < children.length; i++) { const child = children[i] - const leaf = UnixFS.unmarshal(child.node.data) + const leaf = UnixFS.unmarshal(child.node.Data) file.addBlockSize(leaf.fileSize()) - links.push(await DAGLink.create('', child.node.size, child.cid)) + links.push(new DAGLink('', child.node.size, child.cid)) } const node = await DAGNode.create(file.marshal(), links) @@ -129,7 +129,7 @@ describe('exporter', () => { it('ensure hash inputs are sanitized', async () => { const result = await dagPut() const node = await ipld.get(result.cid) - const unmarsh = UnixFS.unmarshal(node.data) + const unmarsh = UnixFS.unmarshal(node.Data) expect(unmarsh.data).to.deep.equal(result.file.data) @@ -183,7 +183,7 @@ describe('exporter', () => { }) const node = await ipld.get(result.cid) - const unmarsh = UnixFS.unmarshal(node.data) + const unmarsh = UnixFS.unmarshal(node.Data) const file = await exporter(result.cid, ipld) const data = Buffer.concat(await all(file.content({ @@ -215,8 +215,8 @@ describe('exporter', () => { file.addBlockSize(5) const fileNode = await DAGNode.create(file.marshal(), [ - await DAGLink.create('', chunkNode1.size, chunkCid1), - await DAGLink.create('', chunkNode2.size, chunkCid2) + new DAGLink('', chunkNode1.size, chunkCid1), + new DAGLink('', chunkNode2.size, chunkCid2) ]) const fileCid = await ipld.put(fileNode, mc.DAG_PB, { cidVersion: 0, @@ -236,7 +236,7 @@ describe('exporter', () => { const result = await dagPut({ content: Buffer.concat(await all(randomBytes(100))), links: [ - await DAGLink.create('', chunk.node.size, chunk.cid) + new DAGLink('', chunk.node.size, chunk.cid) ] }) @@ -671,7 +671,7 @@ describe('exporter', () => { it('fails on non existent hash', async () => { // This hash doesn't exist in the repo - const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKj3' + const hash = 'bafybeidu2qqwriogfndznz32swi5r4p2wruf6ztu5k7my53tsezwhncs5y' try { await exporter(hash, ipld) @@ -840,7 +840,7 @@ describe('exporter', () => { file.addBlockSize(100) const cid = await ipld.put(await DAGNode.create(file.marshal(), [ - await DAGLink.create('', 100, cborNodeCid) + new DAGLink('', 100, cborNodeCid) ]), mc.DAG_PB) const exported = await exporter(cid, ipld) From a193651bfdcbf373218662c4090e00fc83d8f9e3 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 May 2019 21:11:37 +0100 Subject: [PATCH 10/11] chore: PR comments --- .../content/hamt-sharded-directory.js | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js b/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js index 29f22fc..965e1b4 100644 --- a/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js +++ b/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js @@ -1,23 +1,27 @@ 'use strict' const hamtShardedDirectoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => { - return async function * (options = {}) { - const links = node.Links + return (options = {}) => { + return listDirectory(node, path, resolve, depth, ipld, options) + } +} + +async function * listDirectory (node, path, resolve, depth, ipld, options) { + const links = node.Links - for (const link of links) { - const name = link.Name.substring(2) + for (const link of links) { + const name = link.Name.substring(2) - if (name) { - const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, ipld) + if (name) { + const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, ipld) - yield result.entry - } else { - // descend into subshard - node = await ipld.get(link.Hash) + yield result.entry + } else { + // descend into subshard + node = await ipld.get(link.Hash) - for await (const file of hamtShardedDirectoryContent(link.Hash, node, null, path, resolve, depth, ipld)(options)) { - yield file - } + for await (const file of listDirectory(node, path, resolve, depth, ipld, options)) { + yield file } } } From 6a736350fbb36c95e7483bce127f345f27b23639 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 17 May 2019 11:32:10 +0200 Subject: [PATCH 11/11] chore: standardise error codes --- .aegir.js | 7 +++++++ src/index.js | 4 ++-- src/resolvers/dag-cbor.js | 2 +- src/resolvers/index.js | 2 +- src/resolvers/raw.js | 2 +- src/resolvers/unixfs-v1/content/file.js | 2 +- src/resolvers/unixfs-v1/index.js | 2 +- src/utils/validate-offset-and-length.js | 6 +++--- test/exporter.spec.js | 22 +++++++++++----------- 9 files changed, 28 insertions(+), 21 deletions(-) create mode 100644 .aegir.js diff --git a/.aegir.js b/.aegir.js new file mode 100644 index 0000000..e99bbba --- /dev/null +++ b/.aegir.js @@ -0,0 +1,7 @@ +'use strict' + +module.exports = { + karma: { + browserNoActivityTimeout: 1000 * 1000, + } +} diff --git a/src/index.js b/src/index.js index 54a20ad..e5f840f 100644 --- a/src/index.js +++ b/src/index.js @@ -41,7 +41,7 @@ const cidAndRest = (path) => { } } - throw errCode(new Error(`Unknown path type ${path}`), 'EBADPATH') + throw errCode(new Error(`Unknown path type ${path}`), 'ERR_BAD_PATH') } const walkPath = async function * (path, ipld) { @@ -57,7 +57,7 @@ const walkPath = async function * (path, ipld) { const result = await resolve(cid, name, entryPath, toResolve, startingDepth, ipld) if (!result.entry && !result.next) { - throw errCode(new Error(`Could not resolve ${path}`), 'ENOTFOUND') + throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND') } if (result.entry) { diff --git a/src/resolvers/dag-cbor.js b/src/resolvers/dag-cbor.js index 247a2a6..9f95ace 100644 --- a/src/resolvers/dag-cbor.js +++ b/src/resolvers/dag-cbor.js @@ -37,7 +37,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld) => { subObject = subObject[prop] } else { // cannot resolve further - throw errCode(new Error(`No property named ${prop} found in cbor node ${cid.toBaseEncodedString()}`), 'ENOPROP') + throw errCode(new Error(`No property named ${prop} found in cbor node ${cid.toBaseEncodedString()}`), 'ERR_NO_PROP') } } diff --git a/src/resolvers/index.js b/src/resolvers/index.js index bb35964..a3418dc 100644 --- a/src/resolvers/index.js +++ b/src/resolvers/index.js @@ -12,7 +12,7 @@ const resolve = (cid, name, path, toResolve, depth, ipld) => { const resolver = resolvers[cid.codec] if (!resolver) { - throw errCode(new Error(`No resolver for codec ${cid.codec}`), 'ENORESOLVER') + throw errCode(new Error(`No resolver for codec ${cid.codec}`), 'ERR_NO_RESOLVER') } return resolver(cid, name, path, toResolve, resolve, depth, ipld) diff --git a/src/resolvers/raw.js b/src/resolvers/raw.js index dbb95bf..6dc7f68 100644 --- a/src/resolvers/raw.js +++ b/src/resolvers/raw.js @@ -17,7 +17,7 @@ const rawContent = (node) => { const resolve = async (cid, name, path, toResolve, resolve, depth, ipld) => { if (toResolve.length) { - throw errCode(new Error(`No link named ${path} found in raw node ${cid.toBaseEncodedString()}`), 'ENOTFOUND') + throw errCode(new Error(`No link named ${path} found in raw node ${cid.toBaseEncodedString()}`), 'ERR_NOT_FOUND') } const buf = await ipld.get(cid) diff --git a/src/resolvers/unixfs-v1/content/file.js b/src/resolvers/unixfs-v1/content/file.js index 51333db..da4cd9d 100644 --- a/src/resolvers/unixfs-v1/content/file.js +++ b/src/resolvers/unixfs-v1/content/file.js @@ -24,7 +24,7 @@ async function * emitBytes (ipld, node, start, end, streamPosition = 0) { try { file = UnixFS.unmarshal(node.Data) } catch (err) { - throw errCode(err, 'ENOTUNIXFS') + throw errCode(err, 'ERR_NOT_UNIXFS') } // might be a unixfs `raw` node or have data on intermediate nodes diff --git a/src/resolvers/unixfs-v1/index.js b/src/resolvers/unixfs-v1/index.js index b56c66c..49dc33a 100644 --- a/src/resolvers/unixfs-v1/index.js +++ b/src/resolvers/unixfs-v1/index.js @@ -32,7 +32,7 @@ const unixFsResolver = async (cid, name, path, toResolve, resolve, depth, ipld) unixfs = UnixFS.unmarshal(node.Data) } catch (err) { // non-UnixFS dag-pb node? It could happen. - throw errCode(err, 'ENOTUNIXFS') + throw errCode(err, 'ERR_NOT_UNIXFS') } if (!path) { diff --git a/src/utils/validate-offset-and-length.js b/src/utils/validate-offset-and-length.js index 1f207f5..d1eb418 100644 --- a/src/utils/validate-offset-and-length.js +++ b/src/utils/validate-offset-and-length.js @@ -8,11 +8,11 @@ const validateOffsetAndLength = (size, offset, length) => { } if (offset < 0) { - throw errCode(new Error('Offset must be greater than or equal to 0'), 'EINVALIDPARAMS') + throw errCode(new Error('Offset must be greater than or equal to 0'), 'ERR_INVALID_PARAMS') } if (offset > size) { - throw errCode(new Error('Offset must be less than the file size'), 'EINVALIDPARAMS') + throw errCode(new Error('Offset must be less than the file size'), 'ERR_INVALID_PARAMS') } if (!length && length !== 0) { @@ -20,7 +20,7 @@ const validateOffsetAndLength = (size, offset, length) => { } if (length < 0) { - throw errCode(new Error('Length must be greater than or equal to 0'), 'EINVALIDPARAMS') + throw errCode(new Error('Length must be greater than or equal to 0'), 'ERR_INVALID_PARAMS') } if (offset + length > size) { diff --git a/test/exporter.spec.js b/test/exporter.spec.js index ae87120..48459ae 100644 --- a/test/exporter.spec.js +++ b/test/exporter.spec.js @@ -466,7 +466,7 @@ describe('exporter', () => { throw new Error('Should not have got this far') } catch (err) { expect(err.message).to.equal('Length must be greater than or equal to 0') - expect(err.code).to.equal('EINVALIDPARAMS') + expect(err.code).to.equal('ERR_INVALID_PARAMS') } }) @@ -485,7 +485,7 @@ describe('exporter', () => { throw new Error('Should not have got this far') } catch (err) { expect(err.message).to.equal('Offset must be greater than or equal to 0') - expect(err.code).to.equal('EINVALIDPARAMS') + expect(err.code).to.equal('ERR_INVALID_PARAMS') } }) @@ -504,7 +504,7 @@ describe('exporter', () => { throw new Error('Should not have got this far') } catch (err) { expect(err.message).to.equal('Offset must be less than the file size') - expect(err.code).to.equal('EINVALIDPARAMS') + expect(err.code).to.equal('ERR_INVALID_PARAMS') } }) @@ -549,7 +549,7 @@ describe('exporter', () => { throw new Error('Should not have got this far') } catch (err) { expect(err.message).to.contain('Offset must be greater than or equal to 0') - expect(err.code).to.equal('EINVALIDPARAMS') + expect(err.code).to.equal('ERR_INVALID_PARAMS') } }) @@ -590,7 +590,7 @@ describe('exporter', () => { }) } catch (err) { expect(err.message).to.contain('Length must be greater than or equal to 0') - expect(err.code).to.equal('EINVALIDPARAMS') + expect(err.code).to.equal('ERR_INVALID_PARAMS') } }) @@ -602,7 +602,7 @@ describe('exporter', () => { }) } catch (err) { expect(err.message).to.contain('Offset must be less than the file size') - expect(err.code).to.equal('EINVALIDPARAMS') + expect(err.code).to.equal('ERR_INVALID_PARAMS') } }) @@ -786,7 +786,7 @@ describe('exporter', () => { try { await exporter(`${cborNodeCid.toBaseEncodedString()}/baz`, ipld) } catch (err) { - expect(err.code).to.equal('ENOPROP') + expect(err.code).to.equal('ERR_NO_PROP') } }) @@ -807,7 +807,7 @@ describe('exporter', () => { try { await exporter(`${cid.toBaseEncodedString()}`, ipld) } catch (err) { - expect(err.code).to.equal('ENORESOLVER') + expect(err.code).to.equal('ERR_NO_RESOLVER') } }) @@ -817,7 +817,7 @@ describe('exporter', () => { try { await exporter(`${cid.toBaseEncodedString()}/lol`, ipld) } catch (err) { - expect(err.code).to.equal('ENOTFOUND') + expect(err.code).to.equal('ERR_NOT_FOUND') } }) @@ -827,7 +827,7 @@ describe('exporter', () => { try { await exporter(cid, ipld) } catch (err) { - expect(err.code).to.equal('ENOTUNIXFS') + expect(err.code).to.equal('ERR_NOT_UNIXFS') } }) @@ -848,7 +848,7 @@ describe('exporter', () => { try { await all(exported.content()) } catch (err) { - expect(err.code).to.equal('ENOTUNIXFS') + expect(err.code).to.equal('ERR_NOT_UNIXFS') } })