fix: send blobs when running ipfs-http-client in the browser (#3184)

To support streaming of native types with no buffering, normalise add input to blobs and upload using native FormData when the http client is run in the browser.

That is, if the user passes a blob to the http client in the browser leave it alone as enumerating blob contents cause the file data to be read.

Browser FormData objects do not allow you to specify headers for each multipart part which means we can't pass UnixFS metadata via the headers so we turn the metadata into a querystring and append it to the field name for each multipart part as a workaround.

Fixes #3138

BREAKING CHANGES:

- Removes the `mode`, `mtime` and `mtime-nsec` headers from multipart requests
- Passes `mode`, `mtime` and `mtime-nsec` as querystring parameters appended to the field name of multipart requests

Author: achingbrain

package.json

@@ -15,9 +15,10 @@
   ],
   "main": "src/index.js",
   "browser": {
-    "./src/lib/to-stream.js": "./src/lib/to-stream.browser.js",
+    "./src/lib/multipart-request.js": "./src/lib/multipart-request.browser.js",
     "ipfs-utils/src/files/glob-source": false,
-    "go-ipfs": false
+    "go-ipfs": false,
+    "ipfs-core-utils/src/files/normalise-input": "ipfs-core-utils/src/files/normalise-input/index.browser.js"
   },
   "repository": {
     "type": "git",

src/lib/multipart-request.browser.js

@@ -0,0 +1,58 @@
+'use strict'
+
+const normaliseInput = require('ipfs-core-utils/src/files/normalise-input')
+const modeToString = require('./mode-to-string')
+const mtimeToObject = require('./mtime-to-object')
+const { File, FormData } = require('ipfs-utils/src/globalthis')
+
+async function multipartRequest (source = '', abortController, headers = {}) {
+  const formData = new FormData()
+  let index = 0
+
+  for await (const { content, path, mode, mtime } of normaliseInput(source)) {
+    let fileSuffix = ''
+    const type = content ? 'file' : 'dir'
+
+    if (index > 0) {
+      fileSuffix = `-${index}`
+    }
+
+    let fieldName = type + fileSuffix
+    const qs = []
+
+    if (mode !== null && mode !== undefined) {
+      qs.push(`mode=${modeToString(mode)}`)
+    }
+
+    if (mtime != null) {
+      const {
+        secs, nsecs
+      } = mtimeToObject(mtime)
+
+      qs.push(`mtime=${secs}`)
+
+      if (nsecs != null) {
+        qs.push(`mtime-nsecs=${nsecs}`)
+      }
+    }
+
+    if (qs.length) {
+      fieldName = `${fieldName}?${qs.join('&')}`
+    }
+
+    if (content) {
+      formData.set(fieldName, content, encodeURIComponent(path))
+    } else {
+      formData.set(fieldName, new File([''], encodeURIComponent(path), { type: 'application/x-directory' }))
+    }
+
+    index++
+  }
+
+  return {
+    headers,
+    body: formData
+  }
+}
+
+module.exports = multipartRequest

src/lib/multipart-request.js

@@ -1,11 +1,11 @@
 'use strict'
 
 const normaliseInput = require('ipfs-core-utils/src/files/normalise-input')
-const toStream = require('./to-stream')
 const { nanoid } = require('nanoid')
 const modeToString = require('../lib/mode-to-string')
 const mtimeToObject = require('../lib/mtime-to-object')
 const merge = require('merge-options').bind({ ignoreUndefined: true })
+const toStream = require('it-to-stream')
 
 async function multipartRequest (source = '', abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) {
   async function * streamFiles (source) {
@@ -22,26 +22,32 @@ async function multipartRequest (source = '', abortController, headers = {}, bou
           fileSuffix = `-${index}`
         }
 
-        yield `--${boundary}\r\n`
-        yield `Content-Disposition: form-data; name="${type}${fileSuffix}"; filename="${encodeURIComponent(path)}"\r\n`
-        yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n`
+        let fieldName = type + fileSuffix
+        const qs = []
 
         if (mode !== null && mode !== undefined) {
-          yield `mode: ${modeToString(mode)}\r\n`
+          qs.push(`mode=${modeToString(mode)}`)
         }
 
         if (mtime != null) {
           const {
             secs, nsecs
           } = mtimeToObject(mtime)
 
-          yield `mtime: ${secs}\r\n`
+          qs.push(`mtime=${secs}`)
 
           if (nsecs != null) {
-            yield `mtime-nsecs: ${nsecs}\r\n`
+            qs.push(`mtime-nsecs=${nsecs}`)
           }
         }
 
+        if (qs.length) {
+          fieldName = `${fieldName}?${qs.join('&')}`
+        }
+
+        yield `--${boundary}\r\n`
+        yield `Content-Disposition: form-data; name="${fieldName}"; filename="${encodeURIComponent(path)}"\r\n`
+        yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n`
         yield '\r\n'
 
         if (content) {

src/lib/to-stream.browser.js

@@ -0,0 +1,38 @@
+/* eslint-env mocha */
+
+'use strict'
+
+const { Buffer } = require('buffer')
+const { expect } = require('interface-ipfs-core/src/utils/mocha')
+const f = require('./utils/factory')()
+
+describe('.add', function () {
+  this.timeout(20 * 1000)
+
+  let ipfs
+
+  before(async function () {
+    ipfs = (await f.spawn()).api
+  })
+
+  after(() => f.clean())
+
+  it('should ignore metadata until https://github.com/ipfs/go-ipfs/issues/6920 is implemented', async () => {
+    const data = Buffer.from('some data')
+    const result = await ipfs.add(data, {
+      mode: 0o600,
+      mtime: {
+        secs: 1000,
+        nsecs: 0
+      }
+    })
+
+    expect(result).to.not.have.property('mode')
+    expect(result).to.not.have.property('mtime')
+    expect(result).to.have.property('cid')
+
+    const { cid } = result
+    expect(cid).to.have.property('codec', 'dag-pb')
+    expect(cid.toString()).to.equal('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS')
+  })
+})