refactor: add core modules to libp2p

.gitignore

@@ -9,6 +9,7 @@ logs
 *.log
 
 coverage
+.nyc_output
 
 # Runtime data
 pids

examples/pnet-ipfs/.gitignore

@@ -0,0 +1 @@
+tmp/

examples/pnet-ipfs/README.md

@@ -0,0 +1,29 @@
+# Private Networking with IPFS
+This example shows how to set up a private network of IPFS nodes.
+
+## Setup
+Install dependencies:
+
+```
+npm install
+```
+
+## Run
+Running the example will cause two nodes with the same swarm key to be started and exchange basic information.
+
+```
+node index.js
+```
+
+### Using different keys
+This example includes `TASK` comments that can be used to try the example with different swarm keys. This will
+allow you to see how nodes will fail to connect if they are on different private networks and try to connect to
+one another.
+
+To change the swarm key of one of the nodes, look through `index.js` for comments starting with `TASK` to indicate
+where lines are that pertain to changing the swarm key of node 2.
+
+### Exploring the repos
+Once you've run the example you can take a look at the repos in the `./tmp` directory to see how they differ, including
+the swarm keys. You should see a `swarm.key` file in each of the repos and when the nodes are on the same private network
+this contents of the `swarm.key` files should be the same.

examples/pnet-ipfs/index.js

@@ -0,0 +1,145 @@
+/* eslint no-console: ["off"] */
+'use strict'
+
+const IPFS = require('ipfs')
+const assert = require('assert').strict
+const writeKey = require('libp2p-pnet').generate
+const path = require('path')
+const fs = require('fs')
+const privateLibp2pBundle = require('./libp2p-bundle')
+const { mkdirp } = require('./utils')
+
+// Create two separate repo paths so we can run two nodes and check their output
+const repo1 = path.resolve('./tmp', 'repo1', '.ipfs')
+const repo2 = path.resolve('./tmp', 'repo2', '.ipfs')
+mkdirp(repo1)
+mkdirp(repo2)
+
+// Create a buffer and write the swarm key to it
+const swarmKey = Buffer.alloc(95)
+writeKey(swarmKey)
+
+// This key is for the `TASK` mentioned in the writeFileSync calls below
+const otherSwarmKey = Buffer.alloc(95)
+writeKey(otherSwarmKey)
+
+// Add the swarm key to both repos
+const swarmKey1Path = path.resolve(repo1, 'swarm.key')
+const swarmKey2Path = path.resolve(repo2, 'swarm.key')
+fs.writeFileSync(swarmKey1Path, swarmKey)
+// TASK: switch the commented out line below so we're using a different key, to see the nodes fail to connect
+fs.writeFileSync(swarmKey2Path, swarmKey)
+// fs.writeFileSync(swarmKey2Path, otherSwarmKey)
+
+// Create the first ipfs node
+const node1 = new IPFS({
+  repo: repo1,
+  libp2p: privateLibp2pBundle(swarmKey1Path),
+  config: {
+    Addresses: {
+      // Set the swarm address so we dont get port collision on the nodes
+      Swarm: ['/ip4/0.0.0.0/tcp/9101']
+    }
+  }
+})
+
+// Create the second ipfs node
+const node2 = new IPFS({
+  repo: repo2,
+  libp2p: privateLibp2pBundle(swarmKey2Path),
+  config: {
+    Addresses: {
+      // Set the swarm address so we dont get port collision on the nodes
+      Swarm: ['/ip4/0.0.0.0/tcp/9102']
+    }
+  }
+})
+
+console.log('auto starting the nodes...')
+
+// `nodesStarted` keeps track of how many of our nodes have started
+let nodesStarted = 0
+/**
+ * Calls `connectAndTalk` when both nodes have started
+ * @returns {void}
+ */
+const didStartHandler = () => {
+  if (++nodesStarted === 2) {
+    // If both nodes are up, start talking
+    connectAndTalk()
+  }
+}
+
+/**
+ * Exits the process when all started nodes have stopped
+ * @returns {void}
+ */
+const didStopHandler = () => {
+  if (--nodesStarted < 1) {
+    console.log('all nodes stopped, exiting.')
+    process.exit(0)
+  }
+}
+
+/**
+ * Stops the running nodes
+ * @param {Error} err An optional error to log to the console
+ * @returns {void}
+ */
+const doStop = (err) => {
+  if (err) {
+    console.error(err)
+  }
+
+  console.log('Shutting down...')
+  node1.stop()
+  node2.stop()
+}
+
+/**
+ * Connects the IPFS nodes and transfers data between them
+ * @returns {void}
+ */
+const connectAndTalk = async () => {
+  console.log('connecting the nodes...')
+  const node2Id = await node2.id()
+  const dataToAdd = Buffer.from('Hello, private friend!')
+
+  // Connect the nodes
+  // This will error when different private keys are used
+  try {
+    await node1.swarm.connect(node2Id.addresses[0])
+  } catch (err) {
+    return doStop(err)
+  }
+  console.log('the nodes are connected, let\'s add some data')
+
+  // Add some data to node 1
+  let addedCID
+  try {
+    addedCID = await node1.files.add(dataToAdd)
+  } catch (err) {
+    return doStop(err)
+  }
+  console.log(`added ${addedCID[0].path} to the node1`)
+
+  // Retrieve the data from node 2
+  let cattedData
+  try {
+    cattedData = await node2.files.cat(addedCID[0].path)
+  } catch (err) {
+    return doStop(err)
+  }
+  assert.deepEqual(cattedData.toString(), dataToAdd.toString(), 'Should have equal data')
+  console.log(`successfully retrieved "${dataToAdd.toString()}" from node2`)
+
+  doStop()
+}
+
+// Wait for the nodes to boot
+node1.once('start', didStartHandler)
+node2.once('start', didStartHandler)
+
+// Listen for the nodes stopping so we can cleanup
+node1.once('stop', didStopHandler)
+node2.once('stop', didStopHandler)

examples/pnet-ipfs/libp2p-bundle.js

@@ -0,0 +1,60 @@
+'use strict'
+
+const Libp2p = require('libp2p')
+const TCP = require('libp2p-tcp')
+const MPLEX = require('libp2p-mplex')
+const SECIO = require('libp2p-secio')
+const fs = require('fs')
+const Protector = require('libp2p-pnet')
+
+/**
+ * Options for the libp2p bundle
+ * @typedef {Object} libp2pBundle~options
+ * @property {PeerInfo} peerInfo - The PeerInfo of the IPFS node
+ * @property {PeerBook} peerBook - The PeerBook of the IPFS node
+ * @property {Object} config - The config of the IPFS node
+ * @property {Object} options - The options given to the IPFS node
+ */
+
+/**
+ * privateLibp2pBundle returns a libp2p bundle function that will use the swarm
+ * key at the given `swarmKeyPath` to create the Protector
+ *
+ * @param {string} swarmKeyPath The path to our swarm key
+ * @returns {libp2pBundle} Returns a libp2pBundle function for use in IPFS creation
+ */
+const privateLibp2pBundle = (swarmKeyPath) => {
+  /**
+   * This is the bundle we will use to create our fully customized libp2p bundle.
+   *
+   * @param {libp2pBundle~options} opts The options to use when generating the libp2p node
+   * @returns {Libp2p} Our new libp2p node
+   */
+  const libp2pBundle = (opts) => {
+    // Set convenience variables to clearly showcase some of the useful things that are available
+    const peerInfo = opts.peerInfo
+    const peerBook = opts.peerBook
+
+    // Build and return our libp2p node
+    return new Libp2p({
+      peerInfo,
+      peerBook,
+      modules: {
+        transport: [TCP], // We're only using the TCP transport for this example
+        streamMuxer: [MPLEX], // We're only using mplex muxing
+        // Let's make sure to use identifying crypto in our pnet since the protector doesn't
+        // care about node identity, and only the presence of private keys
+        connEncryption: [SECIO],
+        // Leave peer discovery empty, we don't want to find peers. We could omit the property, but it's
+        // being left in for explicit readability.
+        // We should explicitly dial pnet peers, or use a custom discovery service for finding nodes in our pnet
+        peerDiscovery: [],
+        connProtector: new Protector(fs.readFileSync(swarmKeyPath))
+      }
+    })
+  }
+
+  return libp2pBundle
+}
+
+module.exports = privateLibp2pBundle

examples/pnet-ipfs/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "pnet-ipfs-example",
+  "version": "1.0.0",
+  "description": "An example of private networking with IPFS",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node index.js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "ipfs": "~0.32.3",
+    "libp2p": "~0.23.1",
+    "libp2p-mplex": "~0.8.2",
+    "libp2p-pnet": "../../",
+    "libp2p-secio": "~0.10.0",
+    "libp2p-tcp": "~0.13.0"
+  }
+}

examples/pnet-ipfs/utils.js

@@ -0,0 +1,28 @@
+'use strict'
+const fs = require('fs')
+const path = require('path')
+
+/**
+ * mkdirp recursively creates needed folders for the given dir path
+ * @param {string} dir
+ * @returns {string} The path that was created
+ */
+module.exports.mkdirp = (dir) => {
+  return path
+    .resolve(dir)
+    .split(path.sep)
+    .reduce((acc, cur) => {
+      const currentPath = path.normalize(acc + path.sep + cur)
+
+      try {
+        fs.statSync(currentPath)
+      } catch (e) {
+        if (e.code === 'ENOENT') {
+          fs.mkdirSync(currentPath)
+        } else {
+          throw e
+        }
+      }
+      return currentPath
+    }, '')
+}

package.json

@@ -16,7 +16,8 @@
     "test:browser": "aegir test -t browser",
     "release": "aegir release -t node -t browser",
     "release-minor": "aegir release --type minor -t node -t browser",
-    "release-major": "aegir release --type major -t node -t browser"
+    "release-major": "aegir release --type major -t node -t browser",
+    "coverage": "nyc --reporter=text --reporter=lcov npm run test:node"
   },
   "repository": {
     "type": "git",
@@ -44,17 +45,20 @@
   },
   "dependencies": {
     "async": "^2.6.2",
-    "bignumber.js": "^8.1.1",
+    "bignumber.js": "^9.0.0",
     "class-is": "^1.1.0",
     "debug": "^4.1.1",
     "err-code": "^1.1.2",
     "fsm-event": "^2.1.0",
     "hashlru": "^2.3.0",
     "interface-connection": "~0.3.3",
-    "libp2p-circuit": "~0.3.6",
-    "libp2p-connection-manager": "^0.1.0",
-    "libp2p-identify": "~0.7.6",
-    "libp2p-ping": "^0.8.5",
+    "latency-monitor": "~0.2.1",
+    "libp2p-circuit": "./src/circuit",
+    "libp2p-connection-manager": "./src/connection-manager",
+    "libp2p-crypto": "~0.16.1",
+    "libp2p-identify": "./src/identify",
+    "libp2p-ping": "./src/ping",
+    "libp2p-pnet": "./src/pnet",
     "libp2p-switch": "./src/switch",
     "libp2p-websockets": "^0.12.2",
     "mafmt": "^6.0.7",
@@ -65,10 +69,16 @@
     "peer-book": "^0.9.1",
     "peer-id": "^0.12.2",
     "peer-info": "~0.15.1",
-    "pull-stream": "^3.6.13",
+    "pull-cat": "^1.1.11",
+    "pull-defer": "~0.2.3",
+    "pull-handshake": "^1.1.4",
+    "pull-reader": "^1.3.1",
+    "pull-stream": "^3.6.9",
     "promisify-es6": "^1.0.3",
+    "protons": "^1.0.1",
     "retimer": "^2.0.0",
-    "superstruct": "^0.6.0"
+    "superstruct": "^0.6.0",
+    "xsalsa20": "^1.0.2"
   },
   "devDependencies": {
     "@nodeutils/defaults-deep": "^1.1.0",
@@ -80,7 +90,6 @@
     "electron-webrtc": "^0.3.0",
     "interface-datastore": "^0.6.0",
     "libp2p-bootstrap": "^0.9.7",
-    "libp2p-circuit": "^0.3.7",
     "libp2p-delegated-content-routing": "^0.2.2",
     "libp2p-delegated-peer-routing": "^0.2.2",
     "libp2p-floodsub": "~0.17.0",
@@ -103,8 +112,8 @@
     "pull-length-prefixed": "^1.3.3",
     "pull-mplex": "^0.1.2",
     "pull-pair": "^1.1.0",
+    "pull-protocol-buffers": "~0.1.2",
     "pull-serializer": "^0.3.2",
-    "pull-stream": "^3.6.12",
     "sinon": "^7.2.7",
     "webrtcsupport": "^2.2.0",
     "wrtc": "^0.4.1"