diff --git a/package.json b/package.json index 40829a4..741e99b 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "release": "aegir release", "release-minor": "aegir release --type minor", "release-major": "aegir release --type major", - "coverage": "nyc --reporter=text --reporter=lcov npm run test:node", + "coverage": "nyc --reporter=text --reporter=lcov --reporter=html npm run test:node", "dep-check": "aegir dep-check" }, "repository": { @@ -50,6 +50,7 @@ "detect-webworker": "^1.0.0", "dirty-chai": "^2.0.1", "form-data": "^3.0.0", + "ipfs-block": "^0.8.1", "ipfs-block-service": "~0.16.0", "ipfs-repo": "^0.30.1", "ipld": "~0.25.0", @@ -72,11 +73,12 @@ "interface-datastore": "^0.8.0", "ipfs-multipart": "^0.3.0", "ipfs-unixfs": "^0.3.0", - "ipfs-unixfs-exporter": "^0.40.0", - "ipfs-unixfs-importer": "^0.43.1", + "ipfs-unixfs-exporter": "^0.41.0", + "ipfs-unixfs-importer": "^0.44.0", "ipfs-utils": "^0.4.2", "ipld-dag-pb": "^0.18.0", "it-last": "^1.0.1", + "it-pipe": "^1.0.1", "joi-browser": "^13.4.0", "mortice": "^2.0.0", "multicodec": "^1.0.0", diff --git a/src/core/chmod.js b/src/core/chmod.js index bdf71e1..0457342 100644 --- a/src/core/chmod.js +++ b/src/core/chmod.js @@ -12,6 +12,13 @@ const updateMfsRoot = require('./utils/update-mfs-root') const { DAGNode } = require('ipld-dag-pb') const mc = require('multicodec') const mh = require('multihashes') +const pipe = require('it-pipe') +const importer = require('ipfs-unixfs-importer') +const exporter = require('ipfs-unixfs-exporter') +const last = require('it-last') +const cp = require('./cp') +const rm = require('./rm') +const persist = require('ipfs-unixfs-importer/src/utils/persist') const defaultOptions = { flush: true, @@ -21,10 +28,10 @@ const defaultOptions = { recursive: false } -function calculateModification (mode) { +function calculateModification (mode, originalMode, isDirectory) { let modification = 0 - if (mode.includes('x')) { + if (mode.includes('x') || (mode.includes('X') && (isDirectory || (originalMode & 0o1 || originalMode & 0o10 || originalMode & 0o100)))) { modification += 1 } @@ -76,7 +83,7 @@ function calculateSpecial (references, mode, modification) { } // https://en.wikipedia.org/wiki/Chmod#Symbolic_modes -function parseSymbolicMode (input, originalMode) { +function parseSymbolicMode (input, originalMode, isDirectory) { if (!originalMode) { originalMode = 0 } @@ -98,7 +105,7 @@ function parseSymbolicMode (input, originalMode) { references = 'ugo' } - let modification = calculateModification(mode) + let modification = calculateModification(mode, originalMode, isDirectory) modification = calculateUGO(references, modification) modification = calculateSpecial(references, mode, modification) @@ -139,6 +146,20 @@ function parseSymbolicMode (input, originalMode) { } } +function calculateMode (mode, metadata) { + if (typeof mode === 'string' || mode instanceof String) { + if (mode.match(/^\d+$/g)) { + mode = parseInt(mode, 8) + } else { + mode = mode.split(',').reduce((curr, acc) => { + return parseSymbolicMode(acc, curr, metadata.isDirectory()) + }, metadata.mode) + } + } + + return mode +} + module.exports = (context) => { return async function mfsChmod (path, mode, options) { options = applyDefaultOptions(options, defaultOptions) @@ -155,20 +176,54 @@ module.exports = (context) => { throw errCode(new Error(`${path} was not a UnixFS node`), 'ERR_NOT_UNIXFS') } - let node = await context.ipld.get(cid) - const metadata = UnixFS.unmarshal(node.Data) - - if (typeof mode === 'string' || mode instanceof String) { - if (mode.match(/^\d+$/g)) { - mode = parseInt(mode, 8) - } else { - mode = mode.split(',').reduce((curr, acc) => { - return parseSymbolicMode(acc, curr) - }, metadata.mode) - } + if (options.recursive) { + // recursively export from root CID, change perms of each entry then reimport + // but do not reimport files, only manipulate dag-pb nodes + const root = await pipe( + async function * () { + for await (const entry of exporter.recursive(cid, context.ipld)) { + let node = await context.ipld.get(entry.cid) + entry.unixfs.mode = calculateMode(mode, entry.unixfs) + node = new DAGNode(entry.unixfs.marshal(), node.Links) + + yield { + path: entry.path, + content: node + } + } + }, + (source) => importer(source, context.ipld, { + ...options, + dagBuilder: async function * (source, ipld, options) { + for await (const entry of source) { + yield async function () { + const cid = await persist(entry.content, ipld, options) + + return { + cid, + path: entry.path, + unixfs: UnixFS.unmarshal(entry.content.Data), + node: entry.content + } + } + } + } + }), + (nodes) => last(nodes) + ) + + // remove old path from mfs + await rm(context)(path, options) + + // add newly created tree to mfs at path + await cp(context)(`/ipfs/${root.cid}`, path, options) + + return } - metadata.mode = mode + let node = await context.ipld.get(cid) + const metadata = UnixFS.unmarshal(node.Data) + metadata.mode = calculateMode(mode, metadata) node = new DAGNode(metadata.marshal(), node.Links) const updatedCid = await context.ipld.put(node, mc.DAG_PB, { diff --git a/src/core/cp.js b/src/core/cp.js index 2e127aa..19bef3a 100644 --- a/src/core/cp.js +++ b/src/core/cp.js @@ -31,10 +31,6 @@ module.exports = (context) => { throw errCode(new Error('Please supply at least one source'), 'ERR_INVALID_PARAMS') } - if (!destination) { - throw errCode(new Error('Please supply a destination'), 'ERR_INVALID_PARAMS') - } - options.parents = options.p || options.parents // make sure all sources exist diff --git a/src/core/mkdir.js b/src/core/mkdir.js index 155f97d..06be27d 100644 --- a/src/core/mkdir.js +++ b/src/core/mkdir.js @@ -117,8 +117,6 @@ const addEmptyDir = async (context, childName, emptyDir, parent, trail, options) name: childName, hashAlg: options.hashAlg, cidVersion: options.cidVersion, - mode: options.mode, - mtime: options.mtime, flush: options.flush }) diff --git a/src/core/stat.js b/src/core/stat.js index 7330463..2228824 100644 --- a/src/core/stat.js +++ b/src/core/stat.js @@ -74,6 +74,7 @@ const statters = { if (file.unixfs) { output.size = file.unixfs.fileSize() output.type = file.unixfs.type + output.mode = file.unixfs.mode if (file.unixfs.isDirectory()) { output.size = 0 @@ -87,10 +88,6 @@ const statters = { if (file.unixfs.mtime) { output.mtime = file.unixfs.mtime } - - if (file.unixfs.mode !== undefined && file.unixfs.mode !== null) { - output.mode = file.unixfs.mode - } } return output @@ -102,5 +99,17 @@ const statters = { sizeLocal: undefined, withLocality: false } + }, + identity: (file) => { + return { + cid: file.cid, + size: file.node.digest.length, + cumulativeSize: file.node.digest.length, + blocks: 0, + type: 'file', // for go compatibility + local: undefined, + sizeLocal: undefined, + withLocality: false + } } } diff --git a/src/core/utils/add-link.js b/src/core/utils/add-link.js index 431f865..3e4a189 100644 --- a/src/core/utils/add-link.js +++ b/src/core/utils/add-link.js @@ -94,10 +94,14 @@ const addToDirectory = async (context, options) => { options.parent.rmLink(options.name) options.parent.addLink(new DAGLink(options.name, options.size, options.cid)) - // Update mtime const node = UnixFS.unmarshal(options.parent.Data) - node.mtime = new Date() - options.parent = new DAGNode(node.marshal(), options.parent.Links) + + if (node.mtime) { + // Update mtime if previously set + node.mtime = new Date() + + options.parent = new DAGNode(node.marshal(), options.parent.Links) + } const hashAlg = mh.names[options.hashAlg] @@ -110,7 +114,8 @@ const addToDirectory = async (context, options) => { return { node: options.parent, - cid + cid, + size: options.parent.size } } @@ -120,12 +125,13 @@ const addToShardedDirectory = async (context, options) => { } = await addFileToShardedDirectory(context, options) const result = await last(shard.flush('', context.ipld)) + const node = await context.ipld.get(result.cid) // we have written out the shard, but only one sub-shard will have been written so replace it in the original shard const oldLink = options.parent.Links .find(link => link.Name.substring(0, 2) === path[0].prefix) - const newLink = result.node.Links + const newLink = node.Links .find(link => link.Name.substring(0, 2) === path[0].prefix) if (oldLink) { @@ -159,7 +165,11 @@ const addFileToShardedDirectory = async (context, options) => { mode: node.mode }, options) shard._bucket = rootBucket - shard.mtime = new Date() + + if (node.mtime) { + // update mtime if previously set + shard.mtime = new Date() + } // load subshards until the bucket & position no longer changes const position = await rootBucket._findNewBucketAndPos(file.name) diff --git a/src/core/utils/hamt-utils.js b/src/core/utils/hamt-utils.js index 96312bb..31c0522 100644 --- a/src/core/utils/hamt-utils.js +++ b/src/core/utils/hamt-utils.js @@ -34,7 +34,8 @@ const updateHamtDirectory = async (context, links, bucket, options) => { return { node: parent, - cid + cid, + size: parent.size } } diff --git a/src/core/utils/update-tree.js b/src/core/utils/update-tree.js index ed80df4..1eab4c6 100644 --- a/src/core/utils/update-tree.js +++ b/src/core/utils/update-tree.js @@ -47,7 +47,7 @@ const updateTree = async (context, trail, options) => { child = { cid: result.cid, name, - size: result.node.size + size: result.size } } diff --git a/test/core/chmod.js b/test/core/chmod.js index 182f893..f674acd 100644 --- a/test/core/chmod.js +++ b/test/core/chmod.js @@ -95,6 +95,9 @@ describe('chmod', () => { await testChmod('0000', 'ugo+x', '0111') await testChmod('0000', 'ugo+w', '0222') await testChmod('0000', 'ugo+r', '0444') + await testChmod('0000', 'a+x', '0111') + await testChmod('0000', 'a+w', '0222') + await testChmod('0000', 'a+r', '0444') }) it('should update modes with basic symbolic notation that removes bits', async () => { @@ -116,6 +119,9 @@ describe('chmod', () => { await testChmod('0111', 'ugo-x', '0000') await testChmod('0222', 'ugo-w', '0000') await testChmod('0444', 'ugo-r', '0000') + await testChmod('0111', 'a-x', '0000') + await testChmod('0222', 'a-w', '0000') + await testChmod('0444', 'a-r', '0000') }) it('should update modes with basic symbolic notation that overrides bits', async () => { @@ -137,6 +143,9 @@ describe('chmod', () => { await testChmod('0777', 'ugo=x', '0111') await testChmod('0777', 'ugo=w', '0222') await testChmod('0777', 'ugo=r', '0444') + await testChmod('0777', 'a=x', '0111') + await testChmod('0777', 'a=w', '0222') + await testChmod('0777', 'a=r', '0444') }) it('should update modes with multiple symbolic notation', async () => { @@ -149,4 +158,165 @@ describe('chmod', () => { await testChmod('0000', '+t', '1000') await testChmod('0000', '+s', '6000') }) + + it('should apply special execute permissions to world', async () => { + const path = `/foo-${Date.now()}` + const sub = `${path}/sub` + const file = `${path}/sub/foo.txt` + const bin = `${path}/sub/bar` + + await mfs.mkdir(sub, { + parents: true + }) + await mfs.touch(file) + await mfs.touch(bin) + + await mfs.chmod(path, 0o644, { + recursive: true + }) + await mfs.chmod(bin, 'u+x') + + expect((await mfs.stat(path)).mode).to.equal(0o644) + expect((await mfs.stat(sub)).mode).to.equal(0o644) + expect((await mfs.stat(file)).mode).to.equal(0o644) + expect((await mfs.stat(bin)).mode).to.equal(0o744) + + await mfs.chmod(path, 'a+X', { + recursive: true + }) + + // directories should be world-executable + expect((await mfs.stat(path)).mode).to.equal(0o755) + expect((await mfs.stat(sub)).mode).to.equal(0o755) + + // files without prior execute bit should be untouched + expect((await mfs.stat(file)).mode).to.equal(0o644) + + // files with prior execute bit should now be world-executable + expect((await mfs.stat(bin)).mode).to.equal(0o755) + }) + + it('should apply special execute permissions to user', async () => { + const path = `/foo-${Date.now()}` + const sub = `${path}/sub` + const file = `${path}/sub/foo.txt` + const bin = `${path}/sub/bar` + + await mfs.mkdir(sub, { + parents: true + }) + await mfs.touch(file) + await mfs.touch(bin) + + await mfs.chmod(path, 0o644, { + recursive: true + }) + await mfs.chmod(bin, 'u+x') + + expect((await mfs.stat(path)).mode).to.equal(0o644) + expect((await mfs.stat(sub)).mode).to.equal(0o644) + expect((await mfs.stat(file)).mode).to.equal(0o644) + expect((await mfs.stat(bin)).mode).to.equal(0o744) + + await mfs.chmod(path, 'u+X', { + recursive: true + }) + + // directories should be user executable + expect((await mfs.stat(path)).mode).to.equal(0o744) + expect((await mfs.stat(sub)).mode).to.equal(0o744) + + // files without prior execute bit should be untouched + expect((await mfs.stat(file)).mode).to.equal(0o644) + + // files with prior execute bit should now be user executable + expect((await mfs.stat(bin)).mode).to.equal(0o744) + }) + + it('should apply special execute permissions to user and group', async () => { + const path = `/foo-${Date.now()}` + const sub = `${path}/sub` + const file = `${path}/sub/foo.txt` + const bin = `${path}/sub/bar` + + await mfs.mkdir(sub, { + parents: true + }) + await mfs.touch(file) + await mfs.touch(bin) + + await mfs.chmod(path, 0o644, { + recursive: true + }) + await mfs.chmod(bin, 'u+x') + + expect((await mfs.stat(path)).mode).to.equal(0o644) + expect((await mfs.stat(sub)).mode).to.equal(0o644) + expect((await mfs.stat(file)).mode).to.equal(0o644) + expect((await mfs.stat(bin)).mode).to.equal(0o744) + + await mfs.chmod(path, 'ug+X', { + recursive: true + }) + + // directories should be user and group executable + expect((await mfs.stat(path)).mode).to.equal(0o754) + expect((await mfs.stat(sub)).mode).to.equal(0o754) + + // files without prior execute bit should be untouched + expect((await mfs.stat(file)).mode).to.equal(0o644) + + // files with prior execute bit should now be user and group executable + expect((await mfs.stat(bin)).mode).to.equal(0o754) + }) + + it('should apply special execute permissions to sharded directories', async () => { + const path = `/foo-${Date.now()}` + const sub = `${path}/sub` + const file = `${path}/sub/foo.txt` + const bin = `${path}/sub/bar` + + await mfs.mkdir(sub, { + parents: true, + shardSplitThreshold: 0 + }) + await mfs.touch(file, { + shardSplitThreshold: 0 + }) + await mfs.touch(bin, { + shardSplitThreshold: 0 + }) + + await mfs.chmod(path, 0o644, { + recursive: true, + shardSplitThreshold: 0 + }) + await mfs.chmod(bin, 'u+x', { + recursive: true, + shardSplitThreshold: 0 + }) + + expect((await mfs.stat(path)).mode).to.equal(0o644) + expect((await mfs.stat(sub)).mode).to.equal(0o644) + expect((await mfs.stat(file)).mode).to.equal(0o644) + expect((await mfs.stat(bin)).mode).to.equal(0o744) + + await mfs.chmod(path, 'ug+X', { + recursive: true, + shardSplitThreshold: 0 + }) + + // directories should be user and group executable + expect((await mfs.stat(path))).to.include({ + type: 'hamt-sharded-directory', + mode: 0o754 + }) + expect((await mfs.stat(sub)).mode).to.equal(0o754) + + // files without prior execute bit should be untouched + expect((await mfs.stat(file)).mode).to.equal(0o644) + + // files with prior execute bit should now be user and group executable + expect((await mfs.stat(bin)).mode).to.equal(0o754) + }) }) diff --git a/test/core/cp.js b/test/core/cp.js index e22e4e8..a703e4a 100644 --- a/test/core/cp.js +++ b/test/core/cp.js @@ -9,6 +9,9 @@ const createShardedDirectory = require('../helpers/create-sharded-directory') const streamToBuffer = require('../helpers/stream-to-buffer') const streamToArray = require('../helpers/stream-to-array') const crypto = require('crypto') +const CID = require('cids') +const mh = require('multihashes') +const Block = require('ipfs-block') describe('cp', () => { let mfs @@ -53,6 +56,31 @@ describe('cp', () => { } }) + it('refuses to copy files to a non-existent child directory', async () => { + const src1 = `/src2-${Math.random()}` + const src2 = `/src2-${Math.random()}` + const parent = `/output-${Math.random()}` + + await mfs.touch(src1) + await mfs.touch(src2) + await mfs.mkdir(parent) + await expect(mfs.cp(src1, src2, `${parent}/child`)).to.eventually.be.rejectedWith(Error) + .that.has.property('message').that.matches(/destination did not exist/) + }) + + it('refuses to copy files to an unreadable node', async () => { + const src1 = `/src2-${Math.random()}` + const parent = `/output-${Math.random()}` + + const cid = new CID(1, 'identity', mh.encode(Buffer.from('derp'), 'identity')) + await mfs.repo.blocks.put(new Block(Buffer.from('derp'), cid)) + await mfs.cp(`/ipfs/${cid}`, parent) + + await mfs.touch(src1) + await expect(mfs.cp(src1, `${parent}/child`)).to.eventually.be.rejectedWith(Error) + .that.has.property('message').that.matches(/No resolver found for codec "identity"/) + }) + it('refuses to copy files to an exsting file', async () => { const source = `/source-file-${Math.random()}.txt` const destination = `/dest-file-${Math.random()}.txt` diff --git a/test/core/stat.js b/test/core/stat.js index 8b4af83..9517a93 100644 --- a/test/core/stat.js +++ b/test/core/stat.js @@ -8,6 +8,9 @@ const crypto = require('crypto') const createMfs = require('../helpers/create-mfs') const createShardedDirectory = require('../helpers/create-sharded-directory') const mc = require('multicodec') +const CID = require('cids') +const mh = require('multihashes') +const Block = require('ipfs-block') describe('stat', () => { let mfs @@ -172,4 +175,17 @@ describe('stat', () => { expect(stats.cid.toString()).to.equal(cid.toString()) }) + + it('stats an identity CID', async () => { + const data = Buffer.from('derp') + const path = '/identity.node' + const cid = new CID(1, 'identity', mh.encode(data, 'identity')) + await mfs.repo.blocks.put(new Block(data, cid)) + await mfs.cp(`/ipfs/${cid}`, path) + + const stats = await mfs.stat(path) + + expect(stats.cid.toString()).to.equal(cid.toString()) + expect(stats).to.have.property('size', data.length) + }) }) diff --git a/test/helpers/create-mfs.js b/test/helpers/create-mfs.js index f1fc49e..4e25b63 100644 --- a/test/helpers/create-mfs.js +++ b/test/helpers/create-mfs.js @@ -41,6 +41,7 @@ const createMfs = async () => { }) mfs.ipld = ipld + mfs.repo = repo return mfs } diff --git a/test/http/cp.js b/test/http/cp.js index 271ce7f..abbd55a 100644 --- a/test/http/cp.js +++ b/test/http/cp.js @@ -20,7 +20,7 @@ function defaultOptions (modification = {}) { return options } -describe('cp', () => () => { +describe('cp', () => { const source = 'source' const dest = 'dest' let ipfs