feat(blockstore): move to base32 encoding for blocks

This follows go-ipfs, which already made the move. Existing repos can be migrated using fs-repo-migrations.

Closes #67

BREAKING: Changes the repo layout

Author: dignifiedquire

package.json

@@ -36,12 +36,14 @@
     "fs-pull-blob-store": "^0.3.0",
     "idb-pull-blob-store": "^0.3.0",
     "lodash": "^4.15.0",
+    "multihashes": "^0.2.2",
     "ncp": "^2.0.0",
     "pre-commit": "^1.1.3",
     "rimraf": "^2.5.4"
   },
   "dependencies": {
     "babel-runtime": "^6.11.6",
+    "base32.js": "^0.1.0",
     "ipfs-block": "^0.3.0",
     "lock": "^0.1.2",
     "multihashes": "^0.2.2",

src/stores/blockstore.js

@@ -3,18 +3,21 @@
 const Block = require('ipfs-block')
 const pull = require('pull-stream')
 const Lock = require('lock')
+const base32 = require('base32.js')
+const path = require('path')
 
-const PREFIX_LENGTH = 8
+const PREFIX_LENGTH = 5
 
 exports = module.exports
 
 function multihashToPath (multihash, extension) {
   extension = extension || 'data'
-  const filename = `${multihash.toString('hex')}.${extension}`
+  const encoder = new base32.Encoder()
+  const hash = encoder.write(multihash).finalize()
+  const filename = `${hash}.${extension}`
   const folder = filename.slice(0, PREFIX_LENGTH)
-  const path = folder + '/' + filename
 
-  return path
+  return path.join(folder, filename)
 }
 
 exports.setUp = (basePath, BlobStore, locks) => {
@@ -79,8 +82,8 @@ exports.setUp = (basePath, BlobStore, locks) => {
         return cb(new Error('Invalid key'))
       }
 
-      const path = multihashToPath(key, extension)
-      store.exists(path, cb)
+      const p = multihashToPath(key, extension)
+      store.exists(p, cb)
     },
 
     delete (key, extension, cb) {
@@ -93,8 +96,8 @@ exports.setUp = (basePath, BlobStore, locks) => {
         return cb(new Error('Invalid key'))
       }
 
-      const path = multihashToPath(key, extension)
-      store.remove(path, cb)
+      const p = multihashToPath(key, extension)
+      store.remove(p, cb)
     }
   }
 }

test/blockstore-test.js

@@ -3,11 +3,12 @@
 
 const expect = require('chai').expect
 const Block = require('ipfs-block')
+const mh = require('multihashes')
 
 module.exports = (repo) => {
   describe('blockstore', () => {
-    const helloKey = '1220b94d/1220b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9.data'
-    const helloIpldKey = '1220ed12/1220ed12932f3ef94c0792fbc55263968006e867e522cf9faa88274340a2671d4441.ipld'
+    const helloKey = 'CIQLS/CIQLSTJHXGJU2PQIUUXFFV62PWV7VREE57RXUU4A52IIR55M4LX432I.data'
+    const helloIpldKey = 'CIQO2/CIQO2EUTF47PSTAHSL54KUTDS2AAN2DH4URM7H5KRATUGQFCM4OUIQI.ipld'
 
     describe('.put', () => {
       it('simple', (done) => {
@@ -150,5 +151,22 @@ module.exports = (repo) => {
         })
       })
     })
+
+    describe('interop', () => {
+      it('reads welcome-to-ipfs', (done) => {
+        const welcomeHash = mh.fromHexString(
+          '1220120f6af601d46e10b2d2e11ed71c55d25f3042c22501e41d1246e7a1e9d3d8ec'
+        )
+        repo.blockstore.get(welcomeHash, (err, block) => {
+          expect(err).to.not.exist
+          expect(
+            block.data.toString()
+          ).to.match(
+            /Hello and Welcome to IPFS/
+          )
+          done()
+        })
+      })
+    })
   })
 }

test/test-repo/blocks/12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07606bfdb812303d.ext

@@ -1 +1 @@
-MANIFEST-000007
+MANIFEST-000009

test/test-repo/blocks/1220120f/1220120f6af601d46e10b2d2e11ed71c55d25f3042c22501e41d1246e7a1e9d3d8ec.data renamed to test/test-repo/blocks/CIQBE/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data

@@ -1,10 +1,5 @@
-=============== Dec 10, 2015 (PST) ===============
-07:50:02.056578 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
-07:50:02.057231 db@open opening
-07:50:02.057312 journal@recovery F·1
-07:50:02.057514 journal@recovery recovering @3
-07:50:02.058921 mem@flush created L0@5 N·4 S·1KiB "/ip..\xf6\xe4\xa9,v5":"/pk..\xf6\xe4\xa9,v6"
-07:50:02.059983 db@janitor F·4 G·0
-07:50:02.060001 db@open done T·2.755926ms
-07:50:02.073183 db@close closing
-07:50:02.073285 db@close done T·97.522µs
+=============== Aug 16, 2016 (CEST) ===============
+12:27:26.257958 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+12:27:26.258463 db@open opening
+12:27:26.260891 db@janitor F·4 G·0
+12:27:26.260933 db@open done T·2.438178ms

test/test-repo/blocks/122031d6/122031d6da265092f1b03fec969243fdcf095c1d219356cdf538ffce705a52d5738d.data renamed to test/test-repo/blocks/CIQDD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data

@@ -1,10 +1,10 @@
 =============== Dec 10, 2015 (PST) ===============
-07:49:57.048841 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
-07:49:57.049014 db@open opening
-07:49:57.049066 journal@recovery F·1
-07:49:57.049233 journal@recovery recovering @1
-07:49:57.049693 mem@flush created L0@2 N·2 S·211B "/lo..oot,v2":"/lo..ins,v1"
-07:49:57.050381 db@janitor F·3 G·0
-07:49:57.050397 db@open done T·1.375431ms
-07:49:57.064580 db@close closing
-07:49:57.064655 db@close done T·72.59µs
+07:50:02.056578 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+07:50:02.057231 db@open opening
+07:50:02.057312 journal@recovery F·1
+07:50:02.057514 journal@recovery recovering @3
+07:50:02.058921 mem@flush created L0@5 N·4 S·1KiB "/ip..\xf6\xe4\xa9,v5":"/pk..\xf6\xe4\xa9,v6"
+07:50:02.059983 db@janitor F·4 G·0
+07:50:02.060001 db@open done T·2.755926ms
+07:50:02.073183 db@close closing
+07:50:02.073285 db@close done T·97.522µs

test/test-repo/blocks/122031e7/122031e7a41c15d03feb8cd793c3348ea3b310512d7767a9abfbd7a928a85e977173.data renamed to test/test-repo/blocks/CIQDD/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data

@@ -1 +1 @@
-3
+4