feat: upgrade to the new multiformats (#3556)

- Replaces the old [interface-ipld-format](https://github.com/ipld/interface-ipld-format) stack with the new [multiformats](https://github.com/multiformats/js-multiformats) stack.
- The Block API takes/returns `Uint8Array`s instead of [ipld-block](https://github.com/ipld/js-ipld-block) objects

BREAKING CHANGE: ipld-formats no longer supported, use multiformat BlockCodecs instead

Co-authored-by: Rod Vagg <[email protected]>
Co-authored-by: achingbrain <[email protected]>

Author: 3 people

.github/workflows/bundlesize.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [16.x]
         project:
           - packages/ipfs
           - packages/ipfs-core

.github/workflows/typecheck.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [16.x]
         project:
           - packages/ipfs
           - packages/ipfs-cli

.gitignore

@@ -21,6 +21,7 @@ dist
 build
 bundle.js
 tsconfig-types.aegir.json
+tsconfig-check.aegir.json
 .tsbuildinfo
 
 # Deployment files

README.md

@@ -94,20 +94,20 @@ Libp2p `PeerId` instances are no longer returned from the API. If your applicati
 Peer ID strings are also CIDs so converting them is simple:
 
 ```js
-const peerId = PeerId.createFromCID(peerIdStr)
+const peerId = PeerId.createFromB58String(peerIdStr)
 ```
 
 You can get hold of the `PeerId` class using npm or in a script tag:
 
 ```js
 const PeerId = require('peer-id')
-const peerId = PeerId.createFromCID(peerIdStr)
+const peerId = PeerId.createFromB58String(peerIdStr)
 ```
 
 ```html
 <script src="https://unpkg.com/peer-id/dist/index.min.js"></script>
 <script>
-  const peerId = window.PeerId.createFromCID(peerIdStr)
+  const peerId = window.PeerId.createFromB58String(peerIdStr)
 </script>
 ```
 
@@ -120,7 +120,7 @@ Libp2p `PeerInfo` instances are no longer returned from the API. Instead, plain
 Instantiate a new `PeerInfo` and add addresses to it:
 
 ```js
-const peerInfo = new PeerInfo(PeerId.createFromCID(info.id))
+const peerInfo = new PeerInfo(PeerId.createFromB58String(info.id))
 info.addrs.forEach(addr => peerInfo.multiaddrs.add(addr))
 ```
 
@@ -129,15 +129,15 @@ You can get hold of the `PeerInfo` class using npm or in a script tag:
 ```js
 const PeerInfo = require('peer-info')
 const PeerId = require('peer-id')
-const peerInfo = new PeerInfo(PeerId.createFromCID(info.id))
+const peerInfo = new PeerInfo(PeerId.createFromB58String(info.id))
 info.addrs.forEach(addr => peerInfo.multiaddrs.add(addr))
 ```
 
 ```html
 <script src="https://unpkg.com/peer-info/dist/index.min.js"></script>
 <script src="https://unpkg.com/peer-id/dist/index.min.js"></script>
 <script>
-  const peerInfo = new window.PeerInfo(window.PeerId.createFromCID(info.id))
+  const peerInfo = new window.PeerInfo(window.PeerId.createFromB58String(info.id))
   info.addrs.forEach(addr => peerInfo.multiaddrs.add(addr))
 </script>
 ```

docs/MIGRATION-TO-ASYNC-AWAIT.md

@@ -105,9 +105,10 @@ console.log(block.cid.toString())
 // the CID of the object
 
 // With custom format and hashtype through CID
-const CID = require('cids')
+const { CID } = require('multiformats/cid')
+const dagPb = require('@ipld/dag-pb')
 const buf = new TextEncoder().encode('another serialized object')
-const cid = new CID(1, 'dag-pb', multihash)
+const cid = CID.createV1(dagPb.code, multihash)
 
 const block = await ipfs.block.put(blob, cid)
 
@@ -211,7 +212,7 @@ the returned object has the following keys:
 
 ```JavaScript
 const multihashStr = 'QmQULBtTjNcMwMr4VMNknnVv3RpytrLSdgpvMcTnfNhrBJ'
-const cid = new CID(multihashStr)
+const cid = CID.parse(multihashStr)
 
 const stats = await ipfs.block.stat(cid)
 console.log(stats.cid.toString())

docs/core-api/BLOCK.md

@@ -62,7 +62,7 @@ An optional object which may have the following keys:
 
 ```JavaScript
 const obj = { simple: 'object' }
-const cid = await ipfs.dag.put(obj, { format: 'dag-cbor', hashAlg: 'sha3-512' })
+const cid = await ipfs.dag.put(obj, { format: 'dag-cbor', hashAlg: 'sha2-512' })
 
 console.log(cid.toString())
 // zBwWX9ecx5F4X54WAjmFLErnBT6ByfNxStr5ovowTL7AhaUR98RWvXPS1V3HqV1qs3r5Ec5ocv7eCdbqYQREXNUfYNuKG

docs/core-api/DAG.md

@@ -343,7 +343,7 @@ An optional object which may have the following keys:
 const cid = await ipfs.object.patch.addLink(node, {
   name: 'some-link',
   size: 10,
-  cid: new CID('QmPTkMuuL6PD8L2SwTwbcs1NPg14U8mRzerB1ZrrBrkSDD')
+  cid: CID.parse('QmPTkMuuL6PD8L2SwTwbcs1NPg14U8mRzerB1ZrrBrkSDD')
 })
 ```
 
@@ -357,7 +357,7 @@ The `DAGLink` to be added can also be passed as an object containing: `name`, `c
 const link = {
   name: 'Qmef7ScwzJUCg1zUSrCmPAz45m8uP5jU7SLgt2EffjBmbL',
   size: 37,
-  cid: new CID('Qmef7ScwzJUCg1zUSrCmPAz45m8uP5jU7SLgt2EffjBmbL')
+  cid: CID.parse('Qmef7ScwzJUCg1zUSrCmPAz45m8uP5jU7SLgt2EffjBmbL')
 };
 ```
 
@@ -400,7 +400,7 @@ An optional object which may have the following keys:
 const cid = await ipfs.object.patch.rmLink(node, {
   name: 'some-link',
   size: 10,
-  cid: new CID('QmPTkMuuL6PD8L2SwTwbcs1NPg14U8mRzerB1ZrrBrkSDD')
+  cid: CID.parse('QmPTkMuuL6PD8L2SwTwbcs1NPg14U8mRzerB1ZrrBrkSDD')
 })
 ```
 

docs/core-api/OBJECT.md

@@ -86,7 +86,7 @@ An optional object which may have the following keys:
 ### Example
 
 ```JavaScript
-const cid of ipfs.pin.add(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))
+const cid of ipfs.pin.add(CID.parse('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))
 console.log(cid)
 // Logs:
 // CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
@@ -130,7 +130,7 @@ Each yielded object has the form:
 ### Example
 
 ```JavaScript
-for await (const cid of ipfs.pin.addAll(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) {
+for await (const cid of ipfs.pin.addAll(CID.parse('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) {
   console.log(cid)
 }
 // Logs:
@@ -178,7 +178,7 @@ for await (const { cid, type } of ipfs.pin.ls()) {
 
 ```JavaScript
 for await (const { cid, type } of ipfs.pin.ls({
-  paths: [ new CID('Qmc5..'), new CID('QmZb..'), new CID('QmSo..') ]
+  paths: [ CID.parse('Qmc5..'), CID.parse('QmZb..'), CID.parse('QmSo..') ]
 })) {
   console.log({ cid, type })
 }
@@ -218,7 +218,7 @@ An optional object which may have the following keys:
 ### Example
 
 ```JavaScript
-const cid of ipfs.pin.rm(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))
+const cid of ipfs.pin.rm(CID.parse('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))
 console.log(cid)
 // prints the CID that was unpinned
 // CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
@@ -254,7 +254,7 @@ An optional object which may have the following keys:
 ### Example
 
 ```JavaScript
-for await (const cid of ipfs.pin.rmAll(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) {
+for await (const cid of ipfs.pin.rmAll(CID.parse('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) {
   console.log(cid)
 }
 // prints the CIDs that were unpinned
@@ -320,7 +320,7 @@ An object may have the following optional fields:
 
 | Name | Type | Default | Description |
 | ---- | ---- | ------- | ----------- |
-| stat | `boolean` | `false` | If `true` will  include service stats. | 
+| stat | `boolean` | `false` | If `true` will  include service stats. |
 | timeout | `number` | `undefined` | A timeout in ms |
 | signal | [AbortSignal][] | `undefined` |  Can be used to cancel any long running requests started as a result of this call |
 
@@ -486,7 +486,7 @@ Status is one of the following string values:
 ### Example
 
 ```JavaScript
-const cid = new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
+const cid = CID.parse('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
 const pin = await ipfs.pin.remote.add(cid, {
   service: 'pinata',
   name: 'block-party'

docs/core-api/PIN.md

@@ -14,13 +14,14 @@ const main = async () => {
 
   const directoryHash = await streamFiles(ipfs, directoryName, inputFiles)
 
-  const fileList = await ipfs.ls(directoryHash)
-
   log(`\n--\n\nDirectory contents:\n\n${directoryName}/ ${directoryHash}`)
 
-  fileList.forEach((file, index) => {
-    log(` ${index < fileList.length - 1 ? '\u251C' : '\u2514'}\u2500 ${file.name} ${file.path} ${file.hash}`)
-  })
+  let index = 0
+
+  for await (const file of ipfs.ls(directoryHash)) {
+    log(` ${index < inputFiles.length - 1 ? '\u251C' : '\u2514'}\u2500 ${file.name} ${file.path} ${file.cid}`)
+    index++
+  }
 }
 
 const createFiles = (directory) => {
@@ -54,7 +55,7 @@ const streamFiles = async (ipfs, directory, files) => {
 
   const data = await ipfs.add(stream)
 
-  log(`Added ${data.path} hash: ${data.hash}`)
+  log(`Added ${data.path} hash: ${data.cid}`)
 
   // The last data event will contain the directory hash
   if (data.path === directory) {

examples/browser-add-readable-stream/index.js

@@ -15,12 +15,12 @@
     "execa": "^5.0.0",
     "http-server": "^0.12.3",
     "ipfs-http-client": "^50.1.2",
-    "uint8arrays": "^2.1.3"
+    "uint8arrays": "^2.1.6"
   },
   "dependencies": {
     "ipfs": "^0.55.4",
     "it-all": "^1.0.4",
-    "libp2p-websockets": "^0.15.6",
+    "libp2p-websockets": "^0.16.1",
     "rimraf": "^3.0.2",
     "test-ipfs-example": "^3.0.0"
   }

examples/browser-exchange-files/package.json

@@ -8,6 +8,7 @@ const last = require("it-last");
 const cryptoKeys = require("human-crypto-keys"); // { getKeyPairFromSeed }
 const uint8ArrayToString = require('uint8arrays/to-string')
 const uint8ArrayFromString = require('uint8arrays/from-string')
+const { sha256 } = require('multiformats/hashes/sha2')
 
 const { sleep, Logger, onEnterPress, catchAndLog } = require("./util");
 
@@ -142,11 +143,10 @@ async function main() {
     return new Promise(async (resolve, reject) => {
       try {
         // quick and dirty key gen, don't do this in real life
-        const key = await IPFS.multihashing.digest(
+        const key = await sha256.digest(
           uint8ArrayFromString(keyName + Math.random().toString(36).substring(2)),
-          "sha2-256"
         );
-        const keyPair = await cryptoKeys.getKeyPairFromSeed(key, "rsa");
+        const keyPair = await cryptoKeys.getKeyPairFromSeed(key.bytes, "rsa");
 
         // put it on the browser IPNS keychain and name it
         await ipfsBrowser.key.import(keyName, keyPair.privateKey);

examples/browser-ipns-publish/index.js

@@ -16,19 +16,19 @@
     "human-crypto-keys": "^0.1.4",
     "ipfs": "^0.55.4",
     "ipfs-http-client": "^50.1.2",
-    "ipfs-utils": "^8.1.2",
-    "ipns": "^0.12.0",
+    "ipfs-utils": "^8.1.4",
+    "ipns": "^0.13.2",
     "it-last": "^1.0.4",
     "p-retry": "^4.2.0",
-    "uint8arrays": "^2.1.3"
+    "uint8arrays": "^2.1.6"
   },
   "browserslist": [
     "last 2 versions and not dead and > 2%"
   ],
   "devDependencies": {
     "delay": "^5.0.0",
     "execa": "^5.0.0",
-    "ipfsd-ctl": "^8.0.1",
+    "ipfsd-ctl": "^9.0.0",
     "go-ipfs": "0.8.0",
     "parcel": "2.0.0-beta.2",
     "path": "^0.12.7",

examples/browser-ipns-publish/package.json

@@ -17,8 +17,8 @@
     "delay": "^5.0.0",
     "ipfs": "^0.55.4",
     "ipfs-pubsub-room": "^2.0.1",
-    "libp2p-websockets": "^0.15.6",
-    "uint8arrays": "^2.1.3"
+    "libp2p-websockets": "^0.16.1",
+    "uint8arrays": "^2.1.6"
   },
   "devDependencies": {
     "execa": "^5.0.0",